From: John Spray <john.spray@redhat.com>
Date: Wed, 1 Feb 2017 00:25:44 +0000 (+0000)
Subject: tasks/kclient: apply timeout to umount
X-Git-Tag: v12.0.1~448^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3c9f16d8ab4c7c969e47c32d77c82576274d3668;p=ceph.git

tasks/kclient: apply timeout to umount

The umount process can get stuck, in which case
we want to fail the test rather than waiting around for it.

During teardown of the kclient task catch this
timeout explicitly so that we will powercycle the node if
needed.

Signed-off-by: John Spray <john.spray@redhat.com>
---

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 191ea7f4b116..49f11d7c8978 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -7,11 +7,15 @@ from teuthology import misc
 
 from teuthology.orchestra import remote as orchestra_remote
 from teuthology.orchestra import run
+from teuthology.contextutil import MaxWhileTries
 from .mount import CephFSMount
 
 log = logging.getLogger(__name__)
 
 
+UMOUNT_TIMEOUT = 300
+
+
 class KernelMount(CephFSMount):
     def __init__(self, mons, test_dir, client_id, client_remote,
                  ipmi_user, ipmi_password, ipmi_domain):
@@ -96,13 +100,15 @@ class KernelMount(CephFSMount):
 
         self.client_remote.run(args=cmd)
 
-        self.client_remote.run(
+        rproc = self.client_remote.run(
             args=[
                 'rmdir',
                 '--',
                 self.mountpoint,
             ],
+            wait=False
         )
+        run.wait([rproc], UMOUNT_TIMEOUT)
         self.mounted = False
 
     def cleanup(self):
@@ -117,7 +123,7 @@ class KernelMount(CephFSMount):
 
         try:
             self.umount(force)
-        except CommandFailedError:
+        except (CommandFailedError, MaxWhileTries):
             if not force:
                 raise
 
diff --git a/qa/tasks/kclient.py b/qa/tasks/kclient.py
index 454300a2a7c2..7cc7ada35f4a 100644
--- a/qa/tasks/kclient.py
+++ b/qa/tasks/kclient.py
@@ -5,7 +5,9 @@ import contextlib
 import logging
 
 from teuthology.misc import deep_merge
+from teuthology.orchestra.run import CommandFailedError
 from teuthology import misc
+from teuthology.contextutil import MaxWhileTries
 from cephfs.kernel_mount import KernelMount
 
 log = logging.getLogger(__name__)
@@ -113,7 +115,7 @@ def task(ctx, config):
             if mount.is_mounted():
                 try:
                     mount.umount()
-                except CommandFailedError:
+                except (CommandFailedError, MaxWhileTries):
                     log.warn("Ordinary umount failed, forcing...")
                     forced = True
                     mount.umount_wait(force=True)