mgr/cephadm: add keep-alive requests to ssh connections

Fixes: https://tracker.ceph.com/issues/51733
Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 445425ceccaab0cef9c04b795a8fe0236f56d9eb)

diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index 75b3efcec397ea6fdee3fa1811b0925a6d699eb7..6ef71a943e863a9f4bfcd0c219e080282df6835e 100644 (file)
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -75,7 +75,10 @@ class SSHManager:
 
             with self.redirect_log(host, addr):
                 try:
-                    conn = await asyncssh.connect(addr, username=self.mgr.ssh_user, client_keys=[self.mgr.tkey.name], known_hosts=None, config=[self.mgr.ssh_config_fname], preferred_auth=['publickey'])
+                    ssh_options = asyncssh.SSHClientConnectionOptions(keepalive_interval=7, keepalive_count_max=3)
+                    conn = await asyncssh.connect(addr, username=self.mgr.ssh_user, client_keys=[self.mgr.tkey.name],
+                                                  known_hosts=None, config=[self.mgr.ssh_config_fname],
+                                                  preferred_auth=['publickey'], options=ssh_options)
                 except OSError:
                     raise
                 except asyncssh.Error:
@@ -135,9 +138,10 @@ class SSHManager:
         cmd = "sudo " + " ".join(quote(x) for x in cmd)
         logger.debug(f'Running command: {cmd}')
         try:
+            r = await conn.run('sudo true', check=True, timeout=5)
             r = await conn.run(cmd, input=stdin)
         # handle these Exceptions otherwise you might get a weird error like TypeError: __init__() missing 1 required positional argument: 'reason' (due to the asyncssh error interacting with raise_if_exception)
-        except (asyncssh.ChannelOpenError, Exception) as e:
+        except (asyncssh.ChannelOpenError, asyncssh.ProcessError, Exception) as e:
             # SSH connection closed or broken, will create new connection next call
             logger.debug(f'Connection to {host} failed. {str(e)}')
             await self._reset_con(host)