From 478bb3f661621c38ac0b9bb21389cc5b225c318d Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Wed, 25 Mar 2020 18:26:24 +0800 Subject: [PATCH] orchestra/remote.py: use ensure_online() helper in run() for better readability, and to have better errors when we fail to connect to remote. before this change, we could have following backtrace: 2020-03-25T09:00:47.022 INFO:teuthology.lock.ops:Checking smithi088.front.sepia.ceph.com 2020-03-25T09:00:47.023 INFO:teuthology.lock.ops:New key found. Updating... 2020-03-25T09:00:47.044 INFO:teuthology.lock.ops:Updating [smithi088.front.sepia.ceph.com]: set os type and version on server 2020-03-25T09:00:47.044 INFO:teuthology.orchestra.remote:Trying to reconnect to host 2020-03-25T09:00:47.045 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi088.front.sepia.ceph.com', 'timeout': 60} 2020-03-25T09:00:47.046 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2020-03-25T09:00:47.046 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 89, in run_tasks manager.__enter__() File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/internal/lock_machines.py", line 78, in lock_machines os_version, arch) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/lock/ops.py", line 147, in lock_many update_nodes(reimaged) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/lock/ops.py", line 33, in update_nodes inventory_info = remote.inventory_info File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 475, in inventory_info node['arch'] = self.arch File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 459, in arch self._arch = self.sh('uname -m').strip() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 242, in sh proc=self.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 198, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 412, in run transport = client.get_transport() AttributeError: 'NoneType' object has no attribute 'get_transport' Signed-off-by: Kefu Chai --- teuthology/orchestra/remote.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 96a1c920..e0406920 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -162,7 +162,7 @@ class Remote(object): if self.ssh.get_transport() is None: return False try: - self.run(args="true") + self._runner(args="true", client=self.ssh, name=self.shortname) except Exception: return False return self.ssh.get_transport().is_active() @@ -196,10 +196,7 @@ class Remote(object): TODO refactor to move run.run here? """ - if not self.ssh or \ - not self.ssh.get_transport() or \ - not self.ssh.get_transport().is_active(): - self.reconnect() + self.ensure_online() r = self._runner(client=self.ssh, name=self.shortname, **kwargs) r.remote = self return r -- 2.47.3