qa: test cluster logging in DispatchQueue throttling

Fixes: https://tracker.ceph.com/issues/46226
Signed-off-by: Jos Collin <jcollin@redhat.com>

diff --git a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
index e6d6ef99b15d6ac2a6cdc1affe948fd7685ee97f..14dbb474bd015675a659cbcb73d5a764ddba3fbf 100644 (file)
--- a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
+++ b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
@@ -12,3 +12,4 @@ overrides:
       - MDS_CLIENT_LATE_RELEASE
       - responding to mclientcaps
       - RECENT_CRASH
+      - DISPATCH_QUEUE_THROTTLE

diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
index 33748cea5cdce40f1965a4b0f2429740dfaa756c..575a7ba1f1dbb7d826f27b6da0cb7248ad69a78c 100644 (file)
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - DISPATCH_QUEUE_THROTTLE

diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
index fbbe16151ce90eeb56464882caa160550f22b19b..0672ede464739bab9e9d857b248b2712adc15bda 100644 (file)
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - mon_thrash:
     check_mds_failover: True

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
index 33748cea5cdce40f1965a4b0f2429740dfaa756c..575a7ba1f1dbb7d826f27b6da0cb7248ad69a78c 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - DISPATCH_QUEUE_THROTTLE

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
index fbbe16151ce90eeb56464882caa160550f22b19b..0672ede464739bab9e9d857b248b2712adc15bda 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - mon_thrash:
     check_mds_failover: True

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
index 037d399a702d4932e73008d203f10782931cfd5d..759816764b7611182bd8aaaf01d2868b4beed119 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
@@ -5,5 +5,6 @@ overrides:
       - objects unfound and apparently lost
       - MDS_SLOW_METADATA_IO
       - MDS_TRIM
+      - DISPATCH_QUEUE_THROTTLE
 tasks:
 - thrashosds:

diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 14f54a784e793e1a74fca4bd9b41ad514d2c0b52..026e7a170c6e796898a71ae777844186b26c637f 100644 (file)
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -527,6 +527,28 @@ class TestMisc(CephFSTestCase):
             self.run_ceph_cmd('tell', 'cephfs.c', 'something')
         self.assertEqual(ce.exception.exitstatus, 1)
 
+    def test_dispatch_queue_throttle_cluster_log(self):
+        """
+        That cluster log a warning when the Dispatch Queue Throttle Limit hits.
+        """
+        self.config_set('global', 'ms_dispatch_throttle_bytes', 500)
+        self.config_set('global', 'ms_dispatch_throttle_log_interval', 5)
+        # Create files & split across 10 directories, 1000 each.
+        with self.assert_cluster_log("DISPATCH_QUEUE_THROTTLE",
+                                     invert_match=False, watch_channel="cluster"):
+            for i in range(0, 10):
+                self.mount_a.create_n_files("dir{0}/file".format(i), 1000, sync=False)
+
+    def test_dispatch_queue_throttle_health_warn(self):
+        """
+        That a health warning is generated when the Dispatch Queue Throttle Limit hits.
+        """
+        self.config_set('global', 'ms_dispatch_throttle_bytes', 10)
+        self.config_set('global', 'ms_dispatch_throttle_log_interval', 1)
+        # Create files & split across 10 directories, 1000 each.
+        for i in range(0, 10):
+            self.mount_a.create_n_files("dir{0}/file".format(i), 1000, sync=False)
+        self.wait_for_health("DISPATCH_QUEUE_THROTTLE", 120)
 
 @classhook('_add_session_client_evictions')
 class TestSessionClientEvict(CephFSTestCase):