qa: add timeouts for remote ops for client mounts

Fixes: https://tracker.ceph.com/issues/36390
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 092801ae345dfea8f537d1cec2c8dd78519e248a)

diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py
index 8169be57d263cd62c59f29f1534338eecf7709d8..ae4072e480cd1511ebafd9d2dee2b07853f85d05 100644 (file)
--- a/qa/tasks/cephfs/fuse_mount.py
+++ b/qa/tasks/cephfs/fuse_mount.py
@@ -50,6 +50,7 @@ class FuseMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(15*60)
         )
 
         run_cmd = [
@@ -88,12 +89,14 @@ class FuseMount(CephFSMount):
         def list_connections():
             self.client_remote.run(
                 args=["sudo", "mount", "-t", "fusectl", "/sys/fs/fuse/connections", "/sys/fs/fuse/connections"],
-                check_status=False
+                check_status=False,
+                timeout=(15*60)
             )
             p = self.client_remote.run(
                 args=["ls", "/sys/fs/fuse/connections"],
                 stdout=StringIO(),
-                check_status=False
+                check_status=False,
+                timeout=(15*60)
             )
             if p.exitstatus != 0:
                 return []
@@ -163,7 +166,8 @@ class FuseMount(CephFSMount):
             ],
             stdout=StringIO(),
             stderr=StringIO(),
-            wait=False
+            wait=False,
+            timeout=(15*60)
         )
         try:
             proc.wait()
@@ -203,10 +207,10 @@ class FuseMount(CephFSMount):
         # Now that we're mounted, set permissions so that the rest of the test will have
         # unrestricted access to the filesystem mount.
         self.client_remote.run(
-            args=['sudo', 'chmod', '1777', self.mountpoint])
+            args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60))
 
     def _mountpoint_exists(self):
-        return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False).exitstatus == 0
+        return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, timeout=(15*60)).exitstatus == 0
 
     def umount(self):
         try:
@@ -218,6 +222,7 @@ class FuseMount(CephFSMount):
                     '-u',
                     self.mountpoint,
                 ],
+                timeout=(30*60),
             )
         except run.CommandFailedError:
             log.info('Failed to unmount ceph-fuse on {name}, aborting...'.format(name=self.client_remote.name))
@@ -229,7 +234,7 @@ class FuseMount(CephFSMount):
                 run.Raw(';'),
                 'ps',
                 'auxf',
-            ])
+            ], timeout=(60*15))
 
             # abort the fuse mount, killing all hung processes
             if self._fuse_conn:
@@ -252,7 +257,8 @@ class FuseMount(CephFSMount):
                         '-f',
                         self.mountpoint,
                     ],
-                    stderr=stderr
+                    stderr=stderr,
+                    timeout=(60*15)
                 )
             except CommandFailedError:
                 if self.is_mounted():
@@ -307,7 +313,8 @@ class FuseMount(CephFSMount):
                     '--',
                     self.mountpoint,
                 ],
-                stderr=stderr
+                stderr=stderr,
+                timeout=(60*5)
             )
         except CommandFailedError:
             if "No such file or directory" in stderr.getvalue():
@@ -354,6 +361,7 @@ class FuseMount(CephFSMount):
                 '-rf',
                 self.mountpoint,
             ],
+            timeout=(60*5)
         )
 
     def _asok_path(self):
@@ -393,14 +401,14 @@ print find_socket("{client_name}")
         # Find the admin socket
         p = self.client_remote.run(args=[
             'sudo', 'python2', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(15*60))
         asok_path = p.stdout.getvalue().strip()
         log.info("Found client admin socket at {0}".format(asok_path))
 
         # Query client ID from admin socket
         p = self.client_remote.run(
             args=['sudo', self._prefix + 'ceph', '--admin-daemon', asok_path] + args,
-            stdout=StringIO())
+            stdout=StringIO(), timeout=(15*60))
         return json.loads(p.stdout.getvalue())
 
     def get_global_id(self):

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 80271a6ebbb9ea6a7c6785c8817eafcab718a788..4fdbd1b0c5f94d3f7387a7f54d7f4e547e729bd4 100644 (file)
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -43,6 +43,7 @@ class KernelMount(CephFSMount):
                 run.Raw('>'),
                 filename,
             ],
+            timeout=(5*60),
         )
 
     def mount(self, mount_path=None, mount_fs_name=None):
@@ -60,6 +61,7 @@ class KernelMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(5*60),
         )
 
         if mount_path is None:
@@ -84,10 +86,11 @@ class KernelMount(CephFSMount):
                 '-o',
                 opts
             ],
+            timeout=(30*60),
         )
 
         self.client_remote.run(
-            args=['sudo', 'chmod', '1777', self.mountpoint])
+            args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(5*60))
 
         self.mounted = True
 
@@ -99,7 +102,7 @@ class KernelMount(CephFSMount):
             cmd.append('-f')
 
         try:
-            self.client_remote.run(args=cmd, timeout=(5*60))
+            self.client_remote.run(args=cmd, timeout=(15*60))
         except Exception as e:
             self.client_remote.run(args=[
                 'sudo',
@@ -107,7 +110,7 @@ class KernelMount(CephFSMount):
                 'lsof',
                 run.Raw(';'),
                 'ps', 'auxf',
-            ])
+            ], timeout=(15*60))
             raise e
 
         rproc = self.client_remote.run(
@@ -194,6 +197,7 @@ class KernelMount(CephFSMount):
                 '--',
                 self.mountpoint,
             ],
+            timeout=(5*60),
         )
 
     def _find_debug_dir(self):
@@ -219,7 +223,7 @@ class KernelMount(CephFSMount):
 
         p = self.client_remote.run(args=[
             'sudo', 'python', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(5*60))
         client_id_to_dir = json.loads(p.stdout.getvalue())
 
         try:
@@ -241,7 +245,7 @@ class KernelMount(CephFSMount):
 
         p = self.client_remote.run(args=[
             'sudo', 'python', '-c', pyscript
-        ], stdout=StringIO())
+        ], stdout=StringIO(), timeout=(5*60))
         return p.stdout.getvalue()
 
     def get_global_id(self):