qa: add timeouts for remote ops for client mounts

Fixes: https://tracker.ceph.com/issues/36390
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py
index 3443ebcc4154981db8dd2c03adcb0a98185d356d..e76a01650280208ef37de319520bbe7246ca760d 100644 (file)
--- a/qa/tasks/cephfs/fuse_mount.py
+++ b/qa/tasks/cephfs/fuse_mount.py
@@ -56,6 +56,7 @@ class FuseMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(15*60)
         )
 
         run_cmd = [
@@ -94,12 +95,14 @@ class FuseMount(CephFSMount):
         def list_connections():
             self.client_remote.run(
                 args=["sudo", "mount", "-t", "fusectl", "/sys/fs/fuse/connections", "/sys/fs/fuse/connections"],
-                check_status=False
+                check_status=False,
+                timeout=(15*60)
             )
             p = self.client_remote.run(
                 args=["ls", "/sys/fs/fuse/connections"],
                 stdout=StringIO(),
-                check_status=False
+                check_status=False,
+                timeout=(15*60)
             )
             if p.exitstatus != 0:
                 return []
@@ -186,7 +189,8 @@ class FuseMount(CephFSMount):
             ],
             stdout=StringIO(),
             stderr=StringIO(),
-            wait=False
+            wait=False,
+            timeout=(15*60)
         )
         try:
             proc.wait()
@@ -226,10 +230,10 @@ class FuseMount(CephFSMount):
         # Now that we're mounted, set permissions so that the rest of the test will have
         # unrestricted access to the filesystem mount.
         self.client_remote.run(
-            args=['sudo', 'chmod', '1777', self.mountpoint])
+            args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60))
 
     def _mountpoint_exists(self):
-        return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False).exitstatus == 0
+        return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, timeout=(15*60)).exitstatus == 0
 
     def umount(self):
         try:
@@ -241,6 +245,7 @@ class FuseMount(CephFSMount):
                     '-u',
                     self.mountpoint,
                 ],
+                timeout=(30*60),
             )
         except run.CommandFailedError:
             log.info('Failed to unmount ceph-fuse on {name}, aborting...'.format(name=self.client_remote.name))
@@ -252,7 +257,7 @@ class FuseMount(CephFSMount):
                 run.Raw(';'),
                 'ps',
                 'auxf',
-            ])
+            ], timeout=(60*15))
 
             # abort the fuse mount, killing all hung processes
             if self._fuse_conn:
@@ -275,7 +280,8 @@ class FuseMount(CephFSMount):
                         '-f',
                         self.mountpoint,
                     ],
-                    stderr=stderr
+                    stderr=stderr,
+                    timeout=(60*15)
                 )
             except CommandFailedError:
                 if self.is_mounted():
@@ -333,7 +339,8 @@ class FuseMount(CephFSMount):
                     '--',
                     self.mountpoint,
                 ],
-                stderr=stderr
+                stderr=stderr,
+                timeout=(60*5)
             )
         except CommandFailedError:
             if "No such file or directory" in stderr.getvalue():
@@ -382,6 +389,7 @@ class FuseMount(CephFSMount):
                 '-rf',
                 self.mountpoint,
             ],
+            timeout=(60*5)
         )
 
     def _asok_path(self):
@@ -421,14 +429,14 @@ print find_socket("{client_name}")
         # Find the admin socket
         p = self.client_remote.run(args=[
             'sudo', 'python2', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(15*60))
         asok_path = p.stdout.getvalue().strip()
         log.info("Found client admin socket at {0}".format(asok_path))
 
         # Query client ID from admin socket
         p = self.client_remote.run(
             args=['sudo', self._prefix + 'ceph', '--admin-daemon', asok_path] + args,
-            stdout=StringIO())
+            stdout=StringIO(), timeout=(15*60))
         return json.loads(p.stdout.getvalue())
 
     def get_global_id(self):

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index fe6584c7bbe766dcc7106178e8b3ac14f9b07566..6b128f57247d0ed6e82cc621f593bae26c93ebce 100644 (file)
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -43,6 +43,7 @@ class KernelMount(CephFSMount):
                 run.Raw('>'),
                 filename,
             ],
+            timeout=(5*60),
         )
 
     def mount(self, mount_path=None, mount_fs_name=None):
@@ -62,6 +63,7 @@ class KernelMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(5*60),
         )
 
         if mount_path is None:
@@ -86,10 +88,11 @@ class KernelMount(CephFSMount):
                 '-o',
                 opts
             ],
+            timeout=(30*60),
         )
 
         self.client_remote.run(
-            args=['sudo', 'chmod', '1777', self.mountpoint])
+            args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(5*60))
 
         self.mounted = True
 
@@ -101,7 +104,7 @@ class KernelMount(CephFSMount):
             cmd.append('-f')
 
         try:
-            self.client_remote.run(args=cmd, timeout=(5*60))
+            self.client_remote.run(args=cmd, timeout=(15*60))
         except Exception as e:
             self.client_remote.run(args=[
                 'sudo',
@@ -109,7 +112,7 @@ class KernelMount(CephFSMount):
                 'lsof',
                 run.Raw(';'),
                 'ps', 'auxf',
-            ])
+            ], timeout=(15*60))
             raise e
 
         rproc = self.client_remote.run(
@@ -196,6 +199,7 @@ class KernelMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(5*60),
         )
 
     def _find_debug_dir(self):
@@ -221,7 +225,7 @@ class KernelMount(CephFSMount):
 
         p = self.client_remote.run(args=[
             'sudo', 'python', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(5*60))
         client_id_to_dir = json.loads(p.stdout.getvalue())
 
         try:
@@ -243,7 +247,7 @@ class KernelMount(CephFSMount):
 
         p = self.client_remote.run(args=[
             'sudo', 'python', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(5*60))
         return p.stdout.getvalue()
 
     def get_global_id(self):