FOG._wait_for_ready: Tolerate ConnectionResetError

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/provision/fog.py b/teuthology/provision/fog.py
index 319e815495715f34693fc2048a6a47a909660f1a..c30cc760c187ec3d0efa848b899a883725895bd0 100644 (file)
--- a/teuthology/provision/fog.py
+++ b/teuthology/provision/fog.py
@@ -292,7 +292,14 @@ class FOG(object):
         sentinel_file = config.fog.get('sentinel_file', None)
         if sentinel_file:
             cmd = "while [ ! -e '%s' ]; do sleep 5; done" % sentinel_file
-            self.remote.run(args=cmd, timeout=600)
+            action = f"wait for sentinel on {self.shortname}"
+            with safe_while(action=action, timeout=1800, increment=3) as proceed:
+                while proceed():
+                    try:
+                        self.remote.run(args=cmd, timeout=600)
+                        break
+                    except ConnectionResetError as e:
+                        log.error(f"{e} on {self.shortname}")
         self.log.info("Node is ready")
 
     def _fix_hostname(self):