From 2f097219120a96acafead42b14964d9307e424fa Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Wed, 10 Oct 2018 13:33:46 -0700 Subject: [PATCH] qa: add timeouts for remote ops for client mounts Fixes: https://tracker.ceph.com/issues/36390 Signed-off-by: Patrick Donnelly (cherry picked from commit 092801ae345dfea8f537d1cec2c8dd78519e248a) --- qa/tasks/cephfs/fuse_mount.py | 28 ++++++++++++++++++---------- qa/tasks/cephfs/kernel_mount.py | 14 +++++++++----- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py index 8169be57d263c..ae4072e480cd1 100644 --- a/qa/tasks/cephfs/fuse_mount.py +++ b/qa/tasks/cephfs/fuse_mount.py @@ -50,6 +50,7 @@ class FuseMount(CephFSMount): '--', self.mountpoint, ], + timeout=(15*60) ) run_cmd = [ @@ -88,12 +89,14 @@ class FuseMount(CephFSMount): def list_connections(): self.client_remote.run( args=["sudo", "mount", "-t", "fusectl", "/sys/fs/fuse/connections", "/sys/fs/fuse/connections"], - check_status=False + check_status=False, + timeout=(15*60) ) p = self.client_remote.run( args=["ls", "/sys/fs/fuse/connections"], stdout=StringIO(), - check_status=False + check_status=False, + timeout=(15*60) ) if p.exitstatus != 0: return [] @@ -163,7 +166,8 @@ class FuseMount(CephFSMount): ], stdout=StringIO(), stderr=StringIO(), - wait=False + wait=False, + timeout=(15*60) ) try: proc.wait() @@ -203,10 +207,10 @@ class FuseMount(CephFSMount): # Now that we're mounted, set permissions so that the rest of the test will have # unrestricted access to the filesystem mount. self.client_remote.run( - args=['sudo', 'chmod', '1777', self.mountpoint]) + args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60)) def _mountpoint_exists(self): - return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False).exitstatus == 0 + return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, timeout=(15*60)).exitstatus == 0 def umount(self): try: @@ -218,6 +222,7 @@ class FuseMount(CephFSMount): '-u', self.mountpoint, ], + timeout=(30*60), ) except run.CommandFailedError: log.info('Failed to unmount ceph-fuse on {name}, aborting...'.format(name=self.client_remote.name)) @@ -229,7 +234,7 @@ class FuseMount(CephFSMount): run.Raw(';'), 'ps', 'auxf', - ]) + ], timeout=(60*15)) # abort the fuse mount, killing all hung processes if self._fuse_conn: @@ -252,7 +257,8 @@ class FuseMount(CephFSMount): '-f', self.mountpoint, ], - stderr=stderr + stderr=stderr, + timeout=(60*15) ) except CommandFailedError: if self.is_mounted(): @@ -307,7 +313,8 @@ class FuseMount(CephFSMount): '--', self.mountpoint, ], - stderr=stderr + stderr=stderr, + timeout=(60*5) ) except CommandFailedError: if "No such file or directory" in stderr.getvalue(): @@ -354,6 +361,7 @@ class FuseMount(CephFSMount): '-rf', self.mountpoint, ], + timeout=(60*5) ) def _asok_path(self): @@ -393,14 +401,14 @@ print find_socket("{client_name}") # Find the admin socket p = self.client_remote.run(args=[ 'sudo', 'python2', '-c', pyscript - ], stdout=StringIO()) + ], stdout=StringIO(), timeout=(15*60)) asok_path = p.stdout.getvalue().strip() log.info("Found client admin socket at {0}".format(asok_path)) # Query client ID from admin socket p = self.client_remote.run( args=['sudo', self._prefix + 'ceph', '--admin-daemon', asok_path] + args, - stdout=StringIO()) + stdout=StringIO(), timeout=(15*60)) return json.loads(p.stdout.getvalue()) def get_global_id(self): diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py index 80271a6ebbb9e..4fdbd1b0c5f94 100644 --- a/qa/tasks/cephfs/kernel_mount.py +++ b/qa/tasks/cephfs/kernel_mount.py @@ -43,6 +43,7 @@ class KernelMount(CephFSMount): run.Raw('>'), filename, ], + timeout=(5*60), ) def mount(self, mount_path=None, mount_fs_name=None): @@ -60,6 +61,7 @@ class KernelMount(CephFSMount): '--', self.mountpoint, ], + timeout=(5*60), ) if mount_path is None: @@ -84,10 +86,11 @@ class KernelMount(CephFSMount): '-o', opts ], + timeout=(30*60), ) self.client_remote.run( - args=['sudo', 'chmod', '1777', self.mountpoint]) + args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(5*60)) self.mounted = True @@ -99,7 +102,7 @@ class KernelMount(CephFSMount): cmd.append('-f') try: - self.client_remote.run(args=cmd, timeout=(5*60)) + self.client_remote.run(args=cmd, timeout=(15*60)) except Exception as e: self.client_remote.run(args=[ 'sudo', @@ -107,7 +110,7 @@ class KernelMount(CephFSMount): 'lsof', run.Raw(';'), 'ps', 'auxf', - ]) + ], timeout=(15*60)) raise e rproc = self.client_remote.run( @@ -194,6 +197,7 @@ class KernelMount(CephFSMount): '--', self.mountpoint, ], + timeout=(5*60), ) def _find_debug_dir(self): @@ -219,7 +223,7 @@ class KernelMount(CephFSMount): p = self.client_remote.run(args=[ 'sudo', 'python', '-c', pyscript - ], stdout=StringIO()) + ], stdout=StringIO(), timeout=(5*60)) client_id_to_dir = json.loads(p.stdout.getvalue()) try: @@ -241,7 +245,7 @@ class KernelMount(CephFSMount): p = self.client_remote.run(args=[ 'sudo', 'python', '-c', pyscript - ], stdout=StringIO()) + ], stdout=StringIO(), timeout=(5*60)) return p.stdout.getvalue() def get_global_id(self): -- 2.39.5