From: Jos Collin <jcollin@redhat.com>
Date: Wed, 25 Nov 2020 10:08:08 +0000 (+0530)
Subject: qa: test cluster logging in DispatchQueue throttling
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=35dc12185a0fa09fa999324cc2f44ca22b0008e9;p=ceph-ci.git

qa: test cluster logging in DispatchQueue throttling

Fixes: https://tracker.ceph.com/issues/46226
Signed-off-by: Jos Collin <jcollin@redhat.com>
---

diff --git a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
index e6d6ef99b15..14dbb474bd0 100644
--- a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
+++ b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
@@ -12,3 +12,4 @@ overrides:
       - MDS_CLIENT_LATE_RELEASE
       - responding to mclientcaps
       - RECENT_CRASH
+      - DISPATCH_QUEUE_THROTTLE
diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
index 33748cea5cd..575a7ba1f1d 100644
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - DISPATCH_QUEUE_THROTTLE
diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
index fbbe16151ce..0672ede4647 100644
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - mon_thrash:
     check_mds_failover: True
diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
index 33748cea5cd..575a7ba1f1d 100644
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - DISPATCH_QUEUE_THROTTLE
diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
index fbbe16151ce..0672ede4647 100644
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - mon_thrash:
     check_mds_failover: True
diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
index 037d399a702..759816764b7 100644
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
@@ -5,5 +5,6 @@ overrides:
       - objects unfound and apparently lost
       - MDS_SLOW_METADATA_IO
       - MDS_TRIM
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - thrashosds:
diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 14f54a784e7..026e7a170c6 100644
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -527,6 +527,28 @@ class TestMisc(CephFSTestCase):
             self.run_ceph_cmd('tell', 'cephfs.c', 'something')
         self.assertEqual(ce.exception.exitstatus, 1)
 
+    def test_dispatch_queue_throttle_cluster_log(self):
+        """
+        That cluster log a warning when the Dispatch Queue Throttle Limit hits.
+        """
+        self.config_set('global', 'ms_dispatch_throttle_bytes', 500)
+        self.config_set('global', 'ms_dispatch_throttle_log_interval', 5)
+        # Create files & split across 10 directories, 1000 each.
+        with self.assert_cluster_log("DISPATCH_QUEUE_THROTTLE",
+                                     invert_match=False, watch_channel="cluster"):
+            for i in range(0, 10):
+                self.mount_a.create_n_files("dir{0}/file".format(i), 1000, sync=False)
+
+    def test_dispatch_queue_throttle_health_warn(self):
+        """
+        That a health warning is generated when the Dispatch Queue Throttle Limit hits.
+        """
+        self.config_set('global', 'ms_dispatch_throttle_bytes', 10)
+        self.config_set('global', 'ms_dispatch_throttle_log_interval', 1)
+        # Create files & split across 10 directories, 1000 each.
+        for i in range(0, 10):
+            self.mount_a.create_n_files("dir{0}/file".format(i), 1000, sync=False)
+        self.wait_for_health("DISPATCH_QUEUE_THROTTLE", 120)
 
 @classhook('_add_session_client_evictions')
 class TestSessionClientEvict(CephFSTestCase):