qa: Reduce number of osd threads when using compression

Smithi nodes used by teuthology tests have 8 CPU cores and typically run
4 OSD processes. When bluestore software compression is enabled the size
of the OSD thread pool needs to be reduced to 2 threads per OSD because
these threads can easily use 100% of a core. This avoids excessive
amounts of context switches, which leads to OSD threads timing out,
which causes the OSD to drop heartbeat pings and for the monitor to
temporarily mark it down. In extreme cases this can lead to PGs getting
stuck in repeated loops of peering until the teuthology test times out.

Context switches happen oppurtunistically at the end of system calls
so functions with lots of logging are some of the worst affected.

Fixes: https://tracker.ceph.com/issues/72879
Signed-off-by: Bill Scales <bill_scales@uk.ibm.com>

diff --git a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/lz4.yaml b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/lz4.yaml
index 4635ce6f88062e71fb881231653045c6fe2ac931..dae42267d05199689045f15010110756505e65e5 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/lz4.yaml
+++ b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/lz4.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: lz4
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/snappy.yaml b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/snappy.yaml
index e2f4b3eca178a75a77aac3258c5ed0aee61f7ea0..c336c86632ab1509caf314ff4a80412e018b06df 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/snappy.yaml
+++ b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/snappy.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: snappy
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zlib.yaml b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zlib.yaml
index 686e02abddf2465791112ffabc1dcf5bd5315491..8b08227a65bed96af2b86b0933b61e637e208a18 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zlib.yaml
+++ b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zlib.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zlib
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zstd.yaml b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zstd.yaml
index fc37fc18caa4faf5597f8cc97b917b2b51ad2cb8..2235cc8b4082963dfdb692b9405ce83f3060b108 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zstd.yaml
+++ b/qa/objectstore_debug/bluestore/write$/random/compr$/yes$/zstd.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zstd
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/lz4.yaml b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/lz4.yaml
index a565dc4acb199e89f6bb05410720680bd1505a51..8032a751ccd520c57be8000f155ebdf8ff18ea53 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/lz4.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/lz4.yaml
@@ -4,3 +4,5 @@ overrides:
       osd:
         bluestore compression mode: aggressive
         bluestore compression algorithm: lz4
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/snappy.yaml b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/snappy.yaml
index 7e4493bbebc561d3ae308c87c7b22028ba2198a8..b6f92cdacd49e9034235b2624fdb76b3a436004a 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/snappy.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/snappy.yaml
@@ -4,3 +4,5 @@ overrides:
       osd:
         bluestore compression mode: aggressive
         bluestore compression algorithm: snappy
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zlib.yaml b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zlib.yaml
index 83ec6c8b4a78de7655ec2e70fcbf967b54863b58..f07b2f21b1ff055138159a7fa9802a19b2990186 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zlib.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zlib.yaml
@@ -4,3 +4,5 @@ overrides:
       osd:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zlib
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zstd.yaml b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zstd.yaml
index 5149d8aaaf7e80aa93cdcfafee984a7111d8ce3d..9a02ae7c869a25e94e943ed8fc159635d7a1a282 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zstd.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v1/compr$/yes$/zstd.yaml
@@ -4,3 +4,5 @@ overrides:
       osd:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zstd
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/lz4.yaml b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/lz4.yaml
index 4635ce6f88062e71fb881231653045c6fe2ac931..dae42267d05199689045f15010110756505e65e5 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/lz4.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/lz4.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: lz4
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/snappy.yaml b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/snappy.yaml
index e2f4b3eca178a75a77aac3258c5ed0aee61f7ea0..c336c86632ab1509caf314ff4a80412e018b06df 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/snappy.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/snappy.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: snappy
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zlib.yaml b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zlib.yaml
index 686e02abddf2465791112ffabc1dcf5bd5315491..8b08227a65bed96af2b86b0933b61e637e208a18 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zlib.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zlib.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zlib
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1

diff --git a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zstd.yaml b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zstd.yaml
index fc37fc18caa4faf5597f8cc97b917b2b51ad2cb8..2235cc8b4082963dfdb692b9405ce83f3060b108 100644 (file)
--- a/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zstd.yaml
+++ b/qa/objectstore_debug/bluestore/write$/v2/compr$/yes$/zstd.yaml
@@ -5,3 +5,5 @@ overrides:
         bluestore compression mode: aggressive
         bluestore compression algorithm: zstd
         osd_mclock_skip_benchmark: true #new recompression makes us fail the test
+        osd op num threads per shard: 2 #https://tracker.ceph.com/issues/72879
+        osd op num shards: 1