From: Adam King <adking@redhat.com>
Date: Mon, 7 Mar 2022 22:22:56 +0000 (-0500)
Subject: mgr/cephadm: add keep-alive requests to ssh connections
X-Git-Tag: v17.2.1~73^2~55
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8ff2bcf6b0b53c3928c20a67f3da2003f858b3fb;p=ceph.git

mgr/cephadm: add keep-alive requests to ssh connections

Fixes: https://tracker.ceph.com/issues/51733

Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 445425ceccaab0cef9c04b795a8fe0236f56d9eb)
---

diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index 75b3efcec397..6ef71a943e86 100644
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -75,7 +75,10 @@ class SSHManager:
 
             with self.redirect_log(host, addr):
                 try:
-                    conn = await asyncssh.connect(addr, username=self.mgr.ssh_user, client_keys=[self.mgr.tkey.name], known_hosts=None, config=[self.mgr.ssh_config_fname], preferred_auth=['publickey'])
+                    ssh_options = asyncssh.SSHClientConnectionOptions(keepalive_interval=7, keepalive_count_max=3)
+                    conn = await asyncssh.connect(addr, username=self.mgr.ssh_user, client_keys=[self.mgr.tkey.name],
+                                                  known_hosts=None, config=[self.mgr.ssh_config_fname],
+                                                  preferred_auth=['publickey'], options=ssh_options)
                 except OSError:
                     raise
                 except asyncssh.Error:
@@ -135,9 +138,10 @@ class SSHManager:
         cmd = "sudo " + " ".join(quote(x) for x in cmd)
         logger.debug(f'Running command: {cmd}')
         try:
+            r = await conn.run('sudo true', check=True, timeout=5)
             r = await conn.run(cmd, input=stdin)
         # handle these Exceptions otherwise you might get a weird error like TypeError: __init__() missing 1 required positional argument: 'reason' (due to the asyncssh error interacting with raise_if_exception)
-        except (asyncssh.ChannelOpenError, Exception) as e:
+        except (asyncssh.ChannelOpenError, asyncssh.ProcessError, Exception) as e:
             # SSH connection closed or broken, will create new connection next call
             logger.debug(f'Connection to {host} failed. {str(e)}')
             await self._reset_con(host)