From: Xiubo Li <xiubli@redhat.com>
Date: Mon, 6 Jul 2020 12:27:34 +0000 (-0400)
Subject: qa: defer cleaning the mountpoint's netnses and the bridge
X-Git-Tag: wip-pdonnell-testing-20200918.022351~488^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=47600028ae7f1865d9c71a7feee10930bb012f8f;p=ceph-ci.git

qa: defer cleaning the mountpoint's netnses and the bridge

The netnses maybe created/deleted many times in the whole test cases,
we can defer cleaning them untile the last mountpoint is unmounted
or when the test is exiting.

Fixes: https://tracker.ceph.com/issues/46282
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---

diff --git a/qa/tasks/ceph_fuse.py b/qa/tasks/ceph_fuse.py
index 7c651bd6988..f55441a17a6 100644
--- a/qa/tasks/ceph_fuse.py
+++ b/qa/tasks/ceph_fuse.py
@@ -169,3 +169,5 @@ def task(ctx, config):
             mount = info["mount"]
             if mount.is_mounted():
                 mount.umount_wait()
+        for remote in remotes:
+            FuseMount.cleanup_stale_netnses_and_bridge(remote)
diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 8681eeb675b..c72f12a6b4d 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -156,34 +156,39 @@ class CephFSMount(object):
     def _setup_netns(self):
         p = self.client_remote.run(args=['ip', 'netns', 'list'],
                                    stderr=StringIO(), stdout=StringIO(),
-                                   timeout=(5*60))
-        p = p.stdout.getvalue().strip()
-        if re.match(self.netns_name, p) is not None:
-            raise RuntimeError("the netns '{}' already exists!".format(self.netns_name))
+                                   timeout=(5*60)).stdout.getvalue().strip()
 
         # Get the netns name list
         netns_list = re.findall(r'[^()\s][-.\w]+[^():\s]', p)
 
-        # Get an uniq netns id
-        nsid = 0
-        while True:
+        out = re.search(r"{0}".format(self.netns_name), p)
+        if out is None:
+            # Get an uniq nsid for the new netns
+            nsid = 0
             p = self.client_remote.run(args=['ip', 'netns', 'list-id'],
                                        stderr=StringIO(), stdout=StringIO(),
-                                       timeout=(5*60))
-            p = re.search(r"nsid {} ".format(nsid), p.stdout.getvalue())
-            if p is None:
-                break
+                                       timeout=(5*60)).stdout.getvalue()
+            while True:
+                out = re.search(r"nsid {} ".format(nsid), p)
+                if out is None:
+                    break
 
-            nsid += 1
+                nsid += 1
 
-        self.nsid = nsid;
+            # Add one new netns and set it id
+            self.run_shell_payload(f"""
+                set -e
+                sudo ip netns add {self.netns_name}
+                sudo ip netns set {self.netns_name} {nsid}
+            """, timeout=(5*60), omit_sudo=False, cwd='/')
+            self.nsid = nsid;
+        else:
+            # The netns already exists and maybe suspended by self.kill()
+            self.resume_netns();
 
-        # Add one new netns and set it id
-        self.run_shell_payload(f"""
-            set -e
-            sudo ip netns add {self.netns_name}
-            sudo ip netns set {self.netns_name} {nsid}
-        """, timeout=(5*60), omit_sudo=False, cwd='/')
+            nsid = int(re.search(r"{0} \(id: (\d+)\)".format(self.netns_name), p).group(1))
+            self.nsid = nsid;
+            return
 
         # Get one ip address for netns
         ips = IP(self.ceph_brx_net)
@@ -304,9 +309,13 @@ class CephFSMount(object):
         """
         Cleanup the netns for the mountpoint.
         """
-        log.info("Cleaning the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
-        self._cleanup_netns()
-        self._cleanup_brx_and_nat()
+        # We will defer cleaning the netnses and bridge until the last
+        # mountpoint is unmounted, this will be a temporary work around
+        # for issue#46282.
+
+        # log.info("Cleaning the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
+        # self._cleanup_netns()
+        # self._cleanup_brx_and_nat()
 
     def suspend_netns(self):
         """
diff --git a/qa/tasks/kclient.py b/qa/tasks/kclient.py
index 74506e4693e..3917606955a 100644
--- a/qa/tasks/kclient.py
+++ b/qa/tasks/kclient.py
@@ -118,6 +118,9 @@ def task(ctx, config):
                     forced = True
                     mount.umount_wait(force=True)
 
+        for id_, remote in clients:
+            KernelMount.cleanup_stale_netnses_and_bridge(remote)
+
         return forced
 
     ctx.mounts = mounts
diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index 237755d8835..94035d4f28d 100644
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -1480,6 +1480,8 @@ def exec_test():
         verbosity=2,
         failfast=True).run(overall_suite)
 
+    CephFSMount.cleanup_stale_netnses_and_bridge(remote)
+
     if opt_teardown_cluster:
         teardown_cluster()