Use safe_while to work around an OVH problem

We're seeing intermittent network failures when running inside OVH; they
present as:
https://github.com/kennethreitz/requests/issues/2364
This should help work around the issue.

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/lock.py b/teuthology/lock.py
index ae8cd3dbf34c947f8ab1599e412b9b598420337b..38809477afe3d1533c1474f1920b7ef673b1be28 100644 (file)
--- a/teuthology/lock.py
+++ b/teuthology/lock.py
@@ -13,6 +13,7 @@ import teuthology
 from . import misc
 from . import provision
 from .config import config
+from .contextutil import safe_while
 from .lockstatus import get_status
 
 log = logging.getLogger(__name__)
@@ -513,7 +514,14 @@ def unlock_one(ctx, name, user, description=None):
     request = dict(name=name, locked=False, locked_by=user,
                    description=description)
     uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
-    response = requests.put(uri, json.dumps(request))
+    with safe_while(
+            sleep=1, increment=0.5, action="unlock %s" % name) as proceed:
+        while proceed():
+            try:
+                response = requests.put(uri, json.dumps(request))
+            # Work around https://github.com/kennethreitz/requests/issues/2364
+            except requests.exception.ConnectionError as e:
+                log.warn("Saw %s while unlocking; retrying...", str(e))
     success = response.ok
     if success:
         log.info('unlocked %s', name)