mgr/cephadm: Retry command when SSH connection closes

Handle scenarios where the SSH connection is closed or broken. Ensure
commands like exiting maintenance attempt to reconnect rather than
throwing an error to the user.

Fixes: https://tracker.ceph.com/issues/67905
Signed-off-by: Joshua Blanch <joshua.blanch@clyso.com>

diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 4a7959ae04502d735089e65b1ba0f1a6e6996bba..5fc26b10fa86b16a2e01e2dbaf2485aed66a3ecb 100644 (file)
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -1696,7 +1696,12 @@ class CephadmServe:
                 self.log.debug('stdin: %s' % stdin)
 
             cmd = ssh.RemoteCommand(WHICH, ['python3'])
-            python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr)
+            try:
+                # when connection was broken/closed, retrying resets the connection
+                python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr)
+            except ssh.HostConnectionError:
+                python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr)
+
             # N.B. because the python3 executable is based on the results of the
             # which command we can not know it ahead of time and must be converted
             # into a RemoteExecutable.