From b8b62567bc4042ac1da70fc6395f7538ca8e812c Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Thu, 17 Mar 2016 14:56:24 +0100 Subject: [PATCH] orchestra: disable re-keying to fix Key-exchange timed out Disable re-keying entirely: it does not work at all and alway times out. Here is the reproducer: ceph-workbench ceph-qa-suite --verbose --suite upgrade/infernalis-x --suite-branch jewel --ceph jewel --ceph-git-url https://github.com/ceph/ceph --filter 'upgrade:infernalis-x/stress-split/{0-cluster/start.yaml 1-infernalis-install/infernalis.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-mon/mona.yaml 5-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 6-next-mon/monb.yaml 7-workload/{radosbench.yaml rbd_api.yaml} 8-next-mon/monc.yaml 9-workload/{rbd-python.yaml rgw-swift.yaml snaps-many-objects.yaml} distros/centos_7.2.yaml}' Which always fails after ~4h with the following: 2016-03-17T23:44:32.607 INFO:tasks.thrashosds.thrasher:Traceback (most recent call last): File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 639, in wrapper return func(self) File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 693, in do_thrash self.choose_action()() File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 445, in fix_pgp_num self.ceph_manager.set_pool_pgpnum(pool) File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 1294, in set_pool_pgpnum if self.get_num_creating() > 0: File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 1456, in get_num_creating pgs = self.get_pg_stats() File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 1325, in get_pg_stats out = self.raw_cluster_cmd('pg', 'dump', '--format=json') File "/home/ubuntu/src/ceph-qa-suite_master/tasks/ceph_manager.py", line 810, in raw_cluster_cmd stdout=StringIO(), File "/home/ubuntu/teuthology/teuthology/orchestra/remote.py", line 196, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/ubuntu/teuthology/teuthology/orchestra/run.py", line 378, in run r.wait() File "/home/ubuntu/teuthology/teuthology/orchestra/run.py", line 94, in wait greenlet.get() File "/home/ubuntu/teuthology/virtualenv/local/lib/python2.7/site-packages/gevent/greenlet.py", line 274, in get raise self._exception SSHException: Key-exchange timed out waiting for key negotiation http://tracker.ceph.com/issues/15236 Fixes: #15236 Signed-off-by: Loic Dachary --- teuthology/orchestra/monkey.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/teuthology/orchestra/monkey.py b/teuthology/orchestra/monkey.py index cd6104b9c8..9b09c7a8dd 100644 --- a/teuthology/orchestra/monkey.py +++ b/teuthology/orchestra/monkey.py @@ -39,6 +39,12 @@ def patch_100_logger_getChild(): logging.Logger.getChild = getChild +def patch_100_trigger_rekey(): + # Fixes http://tracker.ceph.com/issues/15236 + from paramiko.packet import Packetizer + Packetizer._trigger_rekey = lambda self: True + + def patch_all(): """ Run all the patch_* functions in this module. -- 2.39.5