tasks/kclient: apply timeout to umount

The umount process can get stuck, in which case
we want to fail the test rather than waiting around for it.

During teardown of the kclient task catch this
timeout explicitly so that we will powercycle the node if
needed.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 191ea7f4b1165fde214a93c8d8eb2c01265ea3ad..49f11d7c897898052acd3c898252c12782101839 100644 (file)
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -7,11 +7,15 @@ from teuthology import misc
 
 from teuthology.orchestra import remote as orchestra_remote
 from teuthology.orchestra import run
+from teuthology.contextutil import MaxWhileTries
 from .mount import CephFSMount
 
 log = logging.getLogger(__name__)
 
 
+UMOUNT_TIMEOUT = 300
+
+
 class KernelMount(CephFSMount):
     def __init__(self, mons, test_dir, client_id, client_remote,
                  ipmi_user, ipmi_password, ipmi_domain):
@@ -96,13 +100,15 @@ class KernelMount(CephFSMount):
 
         self.client_remote.run(args=cmd)
 
-        self.client_remote.run(
+        rproc = self.client_remote.run(
             args=[
                 'rmdir',
                 '--',
                 self.mountpoint,
             ],
+            wait=False
         )
+        run.wait([rproc], UMOUNT_TIMEOUT)
         self.mounted = False
 
     def cleanup(self):
@@ -117,7 +123,7 @@ class KernelMount(CephFSMount):
 
         try:
             self.umount(force)
-        except CommandFailedError:
+        except (CommandFailedError, MaxWhileTries):
             if not force:
                 raise
 

diff --git a/qa/tasks/kclient.py b/qa/tasks/kclient.py
index 454300a2a7c254b5477cacccdc14a600b510ddc8..7cc7ada35f4ad3702d7644c109f0a5aacedf6a3d 100644 (file)
--- a/qa/tasks/kclient.py
+++ b/qa/tasks/kclient.py
@@ -5,7 +5,9 @@ import contextlib
 import logging
 
 from teuthology.misc import deep_merge
+from teuthology.orchestra.run import CommandFailedError
 from teuthology import misc
+from teuthology.contextutil import MaxWhileTries
 from cephfs.kernel_mount import KernelMount
 
 log = logging.getLogger(__name__)
@@ -113,7 +115,7 @@ def task(ctx, config):
             if mount.is_mounted():
                 try:
                     mount.umount()
-                except CommandFailedError:
+                except (CommandFailedError, MaxWhileTries):
                     log.warn("Ordinary umount failed, forcing...")
                     forced = True
                     mount.umount_wait(force=True)