From d26e5c228e3cdea4d585bcd077f8513a649ab5d1 Mon Sep 17 00:00:00 2001 From: Chunmei Liu Date: Thu, 27 Nov 2025 07:47:37 +0000 Subject: [PATCH] qa/suites/crimson-rados: add fio test case for osd shard number changes upon restart for 3 osd Signed-off-by: Chunmei Liu --- qa/suites/crimson-rados/osd_shards/% | 0 qa/suites/crimson-rados/osd_shards/.qa | 1 + .../osd_shards/0-crimson_install.yaml | 1 + .../crimson-rados/osd_shards/clusters/.qa | 1 + .../osd_shards/clusters/crimson-fixed.yaml | 26 ++++ .../osd_shards/crimson-supported-all-distro | 1 + .../osd_shards/crimson_qa_overrides.yaml | 1 + qa/suites/crimson-rados/osd_shards/deploy/.qa | 1 + .../crimson-rados/osd_shards/deploy/ceph.yaml | 13 ++ .../osd_shards/deploy/cephadm.yaml.disabled | 16 +++ .../crimson-rados/osd_shards/objectstore/.qa | 1 + .../osd_shards/objectstore/seastore/.qa | 1 + .../seastore/seastore-rbm.yaml.disabled | 0 .../seastore/seastore_segmented.yaml | 1 + qa/suites/crimson-rados/osd_shards/tasks/.qa | 1 + .../osd_shards/tasks/crimson_fio_restart.yaml | 115 ++++++++++++++++++ src/common/options/crimson.yaml.in | 2 +- 17 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 qa/suites/crimson-rados/osd_shards/% create mode 120000 qa/suites/crimson-rados/osd_shards/.qa create mode 120000 qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml create mode 120000 qa/suites/crimson-rados/osd_shards/clusters/.qa create mode 100644 qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml create mode 120000 qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro create mode 120000 qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml create mode 120000 qa/suites/crimson-rados/osd_shards/deploy/.qa create mode 100644 qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml create mode 100644 qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled create mode 120000 qa/suites/crimson-rados/osd_shards/objectstore/.qa create mode 120000 qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa create mode 100644 qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore-rbm.yaml.disabled create mode 120000 qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml create mode 120000 qa/suites/crimson-rados/osd_shards/tasks/.qa create mode 100644 qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml diff --git a/qa/suites/crimson-rados/osd_shards/% b/qa/suites/crimson-rados/osd_shards/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/crimson-rados/osd_shards/.qa b/qa/suites/crimson-rados/osd_shards/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml b/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml new file mode 120000 index 00000000000..e1c3e1bb67f --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml @@ -0,0 +1 @@ +.qa/config/crimson_install.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/clusters/.qa b/qa/suites/crimson-rados/osd_shards/clusters/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/clusters/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml b/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml new file mode 100644 index 00000000000..cac3078ac7e --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml @@ -0,0 +1,26 @@ +overrides: + install: + extra_system_packages: + rpm: + - fio + deb: + - fio + ceph-deploy: + conf: + global: + osd crush chooseleaf type: 0 + osd pool default pg num: 128 + osd pool default pgp num: 128 + ceph: + conf: + osd: + osd shutdown pgref assert: true + global: + ms cluster mode: crc + ms service mode: crc + ms client mode: crc + ms mon service mode: crc + ms mon cluster mode: crc + ms mon client mode: crc +roles: +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] diff --git a/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro b/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro new file mode 120000 index 00000000000..a5b729b9efa --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro @@ -0,0 +1 @@ +.qa/distros/crimson-supported-all-distro/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml b/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml new file mode 120000 index 00000000000..2bf67af1b18 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml @@ -0,0 +1 @@ +.qa/config/crimson_qa_overrides.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/deploy/.qa b/qa/suites/crimson-rados/osd_shards/deploy/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml b/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml new file mode 100644 index 00000000000..1af2f4b1bd2 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml @@ -0,0 +1,13 @@ +tasks: +- install: +- ceph: + conf: + osd: + debug monc: 20 + mon: + mon min osdmap epochs: 50 + paxos service trim min: 10 + # prune full osdmaps regularly + mon osdmap full prune min: 15 + mon osdmap full prune interval: 2 + mon osdmap full prune txsize: 2 diff --git a/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled b/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled new file mode 100644 index 00000000000..0c2062240ee --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled @@ -0,0 +1,16 @@ +# no need to verify os + flavor + sha1 +verify_ceph_hash: false +tasks: +- cephadm: + conf: + mgr: + debug ms: 1 + debug mgr: 20 + debug osd: 10 +- cephadm.shell: + mon.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/.qa b/qa/suites/crimson-rados/osd_shards/objectstore/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore-rbm.yaml.disabled b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore-rbm.yaml.disabled new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml new file mode 120000 index 00000000000..46903a8c942 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml @@ -0,0 +1 @@ +.qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/tasks/.qa b/qa/suites/crimson-rados/osd_shards/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml b/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml new file mode 100644 index 00000000000..6a7b3f07c00 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml @@ -0,0 +1,115 @@ +overrides: + ceph: + log-ignorelist: + - PG_DEGRADED + - OSD_DOWN + - OSD_HOST_DOWN + - OSD_ROOT_DOWN + - PG_AVAILABILITY +tasks: +- exec: + mon.a: + - ceph -s + +- exec: + client.0: + - | + set -ex + if ! ceph osd pool ls | grep -qx 'testpool'; then + ceph osd pool create testpool 64 64 + fi + ceph osd pool application enable testpool rbd || true + ceph -s + if ! rbd info testpool/testimage; then + rbd create testpool/testimage --size 10G --image-format=2 + fi +- exec: + client.0: + - | + set -ex + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards + ceph config set osd seastore_require_partition_count_match_reactor_count false + ceph config set osd.0 crimson_cpu_num 2 + ceph config set osd.1 crimson_cpu_num 2 + ceph config set osd.2 crimson_cpu_num 2 + +- ceph.restart: + daemons: [osd.0, osd.1, osd.2] + wait-for-healthy: false + wait-for-osds-up: true + +- exec: + osd.0: + - | + set -ex + ceph health detail + ceph daemon osd.0 config show +- exec: + client.0: + - | + set -ex + rbd info testpool/testimage + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards + ceph config set osd.0 crimson_cpu_num 5 + ceph config set osd.1 crimson_cpu_num 5 + ceph config set osd.2 crimson_cpu_num 5 +- ceph.restart: + daemons: [osd.0, osd.1, osd.2] + wait-for-healthy: false + wait-for-osds-up: true + +- exec: + osd.0: + - | + set -ex + ceph health detail + ceph daemon osd.0 config show + +- exec: + client.0: + - | + set -ex + rbd info testpool/testimage + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards \ No newline at end of file diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in index bce550643fc..24e080beacd 100644 --- a/src/common/options/crimson.yaml.in +++ b/src/common/options/crimson.yaml.in @@ -18,7 +18,7 @@ options: - name: crimson_cpu_num type: uint level: advanced - default: 0 + default: 3 desc: The number of CPUs to use for serving seastar reactors per OSD without CPU pinning. long_desc: The number of CPUs to use for serving seastar reactors per OSD without CPU pinning. overridden if crimson_cpu_set is set. -- 2.47.3