From: Zack Cerza <zack@redhat.com>
Date: Mon, 19 Oct 2015 21:35:37 +0000 (-0600)
Subject: Use safe_while to work around an OVH problem
X-Git-Tag: 1.1.0~791^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F659%2Fhead;p=teuthology.git

Use safe_while to work around an OVH problem

We're seeing intermittent network failures when running inside OVH; they
present as:
https://github.com/kennethreitz/requests/issues/2364
This should help work around the issue.

Signed-off-by: Zack Cerza <zack@redhat.com>
---

diff --git a/teuthology/lock.py b/teuthology/lock.py
index ae8cd3db..38809477 100644
--- a/teuthology/lock.py
+++ b/teuthology/lock.py
@@ -13,6 +13,7 @@ import teuthology
 from . import misc
 from . import provision
 from .config import config
+from .contextutil import safe_while
 from .lockstatus import get_status
 
 log = logging.getLogger(__name__)
@@ -513,7 +514,14 @@ def unlock_one(ctx, name, user, description=None):
     request = dict(name=name, locked=False, locked_by=user,
                    description=description)
     uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
-    response = requests.put(uri, json.dumps(request))
+    with safe_while(
+            sleep=1, increment=0.5, action="unlock %s" % name) as proceed:
+        while proceed():
+            try:
+                response = requests.put(uri, json.dumps(request))
+            # Work around https://github.com/kennethreitz/requests/issues/2364
+            except requests.exception.ConnectionError as e:
+                log.warn("Saw %s while unlocking; retrying...", str(e))
     success = response.ok
     if success:
         log.info('unlocked %s', name)