ceph_manager: eventually time out while waiting for admin socket

Otherwise we get stuck in a loop if an osd crashes unexpectedly, the
task never fails, and we don't collect all the evidence.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py
index 46a4d1d51cf36d2fb87380052d2077ed92d186d2..2e22bc14658b669c15a69f7fee23ed453177268f 100644 (file)
--- a/teuthology/task/ceph_manager.py
+++ b/teuthology/task/ceph_manager.py
@@ -316,6 +316,7 @@ class CephManager:
             '0')
 
     def wait_run_admin_socket(self, osdnum, args=['version']):
+        tries = 0
         while True:
             proc = self.osd_admin_socket(
                 osdnum, args,
@@ -323,11 +324,15 @@ class CephManager:
             if proc.exitstatus is 0:
                 break
             else:
+                tries += 1
+                if tries > 15:
+                    raise 'timed out waiting for admin_socket to appear after osd.{o} restart'.format(o=osdnum)
                 self.log(
                     "waiting on admin_socket for {osdnum}, {command}".format(
                         osdnum=osdnum,
                         command=args))
                 time.sleep(5)
+                max += 1
 
     def set_config(self, osdnum, **argdict):
         for k,v in argdict.iteritems():