qa: test DispatchQueue throttling

Fixes: https://tracker.ceph.com/issues/46226
Signed-off-by: Jos Collin <jcollin@redhat.com>

diff --git a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
index 40d63ba792b11ea5110d73e8de0cff99488bc55d..d9d5d3ca39d5dc5a970a3bed785196a828c3727c 100644 (file)
--- a/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
+++ b/qa/suites/fs/multiclient/tasks/cephfs_misc_tests.yaml
@@ -11,3 +11,4 @@ overrides:
       - has not responded to cap revoke by MDS for over
       - MDS_CLIENT_LATE_RELEASE
       - responding to mclientcaps
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.

diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
index 33748cea5cdce40f1965a4b0f2429740dfaa756c..a151d7a49cf0f68e164ecc7dfdadde3c77376838 100644 (file)
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.

diff --git a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
index fbbe16151ce90eeb56464882caa160550f22b19b..370fd66f79fb8c817f468e65fd66312c7f541b72 100644 (file)
--- a/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/multifs/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.
 tasks:
 - mon_thrash:
     check_mds_failover: True

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
index 33748cea5cdce40f1965a4b0f2429740dfaa756c..a151d7a49cf0f68e164ecc7dfdadde3c77376838 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mds.yaml
@@ -5,3 +5,4 @@ overrides:
   ceph:
     log-ignorelist:
       - Replacing daemon mds
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
index fbbe16151ce90eeb56464882caa160550f22b19b..370fd66f79fb8c817f468e65fd66312c7f541b72 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/mon.yaml
@@ -3,6 +3,7 @@ overrides:
     log-ignorelist:
       - overall HEALTH_
       - \(MON_DOWN\)
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.
 tasks:
 - mon_thrash:
     check_mds_failover: True

diff --git a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
index 62e30ba230fa4c94fa3c882116c10356b053aea9..968ac782d5b4d205742487f7d199446edc31b4be 100644 (file)
--- a/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
+++ b/qa/suites/fs/thrash/workloads/tasks/1-thrash/osd.yaml
@@ -4,5 +4,6 @@ overrides:
       - but it is still running
       - objects unfound and apparently lost
       - MDS_SLOW_METADATA_IO
+      - Throttler Limit has been hit. Some message processing may be significantly delayed.
 tasks:
 - thrashosds:

diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 6a295bbfdf1aca7dcf442385b994c1e3e4e69c6e..6aa222f9a617733afe996e7e8f1db1d8ccc81af8 100644 (file)
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -198,6 +198,18 @@ class TestMisc(CephFSTestCase):
         info = self.fs.mds_asok(['dump', 'inode', hex(ino)])
         assert info['path'] == "/foo"
 
+    def test_dispatch_queue_throttle_message(self):
+        """
+        That cluster log a warning when the Dispatch Queue Throttle Limit hits
+        """
+        self.config_set('mds', 'ms_dispatch_throttle_log_interval', 5)
+        self.config_set('mds', 'ms_dispatch_throttle_bytes', 10240)
+
+        # Create files & split across 10 directories, 1000 each.
+        with self.assert_cluster_log("Throttler Limit has been hit. Some message processing may be significantly delayed.",
+                                     invert_match=False, watch_channel="cluster"):
+            for i in range(0, 10):
+                self.mount_a.create_n_files("dir{0}/file".format(i), 1000, sync=False)
 
 class TestCacheDrop(CephFSTestCase):
     CLIENTS_REQUIRED = 1