client: workaround socket leak

Recreate the boto connection object after every 512 requests in an
attempt to workaround a socket leak when the remote end closes the
connection. Boto does not seem to be causing httplib to close() the
sockets appropriately in some cases. I'm not sure exactly where
the leak is occurring, but forcing boto to reinitialize its
connection pool like this avoids it.

Signed-off-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/radosgw_agent/client.py b/radosgw_agent/client.py
index 42a03ec90080f96906a8ef09954f6bef547b4b1e..c08788bc6f7b99fcc4b890d50dfbb5361eec476f 100644 (file)
--- a/radosgw_agent/client.py
+++ b/radosgw_agent/client.py
@@ -117,7 +117,8 @@ def request(connection, type_, resource, params=None, headers=None,
     if params is None:
         params = {}
     safe_params = dict([(k, url_safe(v)) for k, v in params.iteritems()])
-    request = aws_request.base_http_request(connection,
+    connection.count_request()
+    request = aws_request.base_http_request(connection.s3_connection,
                              type_.upper(),
                              resource=resource,
                              special_first_param=special_first_param,
@@ -135,7 +136,7 @@ def request(connection, type_, resource, params=None, headers=None,
                    url, params, request.headers, data)
     try:
         result = aws_request.make_request(
-            connection,
+            connection.s3_connection,
             type_.upper(),
             resource=resource,
             special_first_param=special_first_param,
@@ -480,13 +481,33 @@ def configure_endpoints(region_map, dest_endpoint, src_endpoint, meta_only):
     src_endpoint.region = src_region
     src_endpoint.zone = src_zone
 
-def connection(endpoint, debug=None):
-    return S3Connection(
-        aws_access_key_id=endpoint.access_key,
-        aws_secret_access_key=endpoint.secret_key,
-        is_secure=endpoint.secure,
-        host=endpoint.host,
-        port=endpoint.port,
-        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
-        debug=debug,
+class S3ConnectionWrapper(object):
+    def __init__(self, endpoint, debug):
+        self.endpoint = endpoint
+        self.debug = debug
+        self.s3_connection = None
+        self.reqs_before_reset = 512
+        self._recreate_s3_connection()
+
+    def count_request(self):
+        self.num_requests += 1
+        if self.num_requests > self.reqs_before_reset:
+            self._recreate_s3_connection()
+
+    def _recreate_s3_connection(self):
+        self.num_requests = 0
+        self.s3_connection = S3Connection(
+            aws_access_key_id=self.endpoint.access_key,
+            aws_secret_access_key=self.endpoint.secret_key,
+            is_secure=self.endpoint.secure,
+            host=self.endpoint.host,
+            port=self.endpoint.port,
+            calling_format=boto.s3.connection.OrdinaryCallingFormat(),
+            debug=self.debug,
         )
+
+    def __getattr__(self, attrib):
+        return getattr(self.s3_connection, attrib)
+
+def connection(endpoint, debug=None):
+    return S3ConnectionWrapper(endpoint, debug)

diff --git a/radosgw_agent/tests/test_client.py b/radosgw_agent/tests/test_client.py
index 08821fcaa2591e317b5199d5e851ac75006aed68..521aa2c92c2a78bd5119eb54ebbcfc50483af666 100644 (file)
--- a/radosgw_agent/tests/test_client.py
+++ b/radosgw_agent/tests/test_client.py
@@ -392,14 +392,9 @@ class TestRequest(object):
             body='{}',
             content_type="application/json",
         )
-        connection = client.S3Connection(
-            aws_access_key_id='key',
-            aws_secret_access_key='secret',
-            is_secure=False,
-            host='localhost',
-            port=8888,
-            calling_format=client.boto.s3.connection.OrdinaryCallingFormat(),
-            debug=True,
+        connection = client.connection(
+            client.Endpoint('localhost', 8888, False, 'key', 'secret'),
+            True,
         )
 
         client.request(connection, 'get', '/%7E~', _retries=0)
@@ -415,14 +410,9 @@ class TestRequest(object):
             body='{"msg": "ok"}',
             content_type="application/json",
         )
-        connection = client.S3Connection(
-            aws_access_key_id='key',
-            aws_secret_access_key='secret',
-            is_secure=False,
-            host='localhost',
-            port=8888,
-            calling_format=client.boto.s3.connection.OrdinaryCallingFormat(),
-            debug=True,
+        connection = client.connection(
+            client.Endpoint('localhost', 8888, False, 'key', 'secret'),
+            True,
         )
 
         result = client.request(connection, 'get', '/%7E~', _retries=0)
@@ -438,14 +428,9 @@ class TestRequest(object):
             content_type="application/json",
             status=500,
         )
-        connection = client.S3Connection(
-            aws_access_key_id='key',
-            aws_secret_access_key='secret',
-            is_secure=False,
-            host='localhost',
-            port=8888,
-            calling_format=client.boto.s3.connection.OrdinaryCallingFormat(),
-            debug=True,
+        connection = client.connection(
+            client.Endpoint('localhost', 8888, False, 'key', 'secret'),
+            True,
         )
 
         with py.test.raises(client.HttpError):
@@ -481,14 +466,9 @@ class TestBotoCall(object):
 class TestGETClientRequests(object):
 
     def setup(self):
-        self.connection = client.S3Connection(
-            aws_access_key_id='key',
-            aws_secret_access_key='secret',
-            is_secure=False,
-            host='localhost',
-            port=8888,
-            calling_format=client.boto.s3.connection.OrdinaryCallingFormat(),
-            debug=True,
+        self.connection = client.connection(
+            client.Endpoint('localhost', 8888, False, 'key', 'secret'),
+            True,
         )
 
     def register(self):
@@ -545,15 +525,5 @@ class TestGETClientRequests(object):
             body='{"msg": "ok"}',
             content_type="application/json",
         )
-        connection = client.S3Connection(
-            aws_access_key_id='key',
-            aws_secret_access_key='secret',
-            is_secure=False,
-            host='localhost',
-            port=8888,
-            calling_format=client.boto.s3.connection.OrdinaryCallingFormat(),
-            debug=True,
-        )
-
-        result = client.request(connection, 'get', '/%7E~')
+        result = client.request(self.connection, 'get', '/%7E~')
         assert result == {'msg': 'ok'}