From: Chunmei Liu Date: Thu, 27 Nov 2025 07:47:37 +0000 (+0000) Subject: qa/suites/crimson-rados: add fio test case for osd shard number changes upon restart... X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F64975%2Fhead;p=ceph.git qa/suites/crimson-rados: add fio test case for osd shard number changes upon restart for 3 osd Signed-off-by: Chunmei Liu --- diff --git a/qa/suites/crimson-rados/osd_shards/% b/qa/suites/crimson-rados/osd_shards/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/crimson-rados/osd_shards/.qa b/qa/suites/crimson-rados/osd_shards/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml b/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml new file mode 120000 index 00000000000..e1c3e1bb67f --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/0-crimson_install.yaml @@ -0,0 +1 @@ +.qa/config/crimson_install.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/clusters/.qa b/qa/suites/crimson-rados/osd_shards/clusters/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/clusters/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml b/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml new file mode 100644 index 00000000000..75702a9c41d --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/clusters/crimson-fixed.yaml @@ -0,0 +1,31 @@ +overrides: + install: + extra_system_packages: + rpm: + - fio + deb: + - fio + ceph: + conf: + global: + osd crush chooseleaf type: 0 + osd pool default pg num: 128 + osd pool default pgp num: 128 + ms cluster mode: crc + ms service mode: crc + ms client mode: crc + ms mon service mode: crc + ms mon cluster mode: crc + ms mon client mode: crc + osd: + osd shutdown pgref assert: true + debug monc: 20 + mon: + mon min osdmap epochs: 50 + paxos service trim min: 10 + # prune full osdmaps regularly + mon osdmap full prune min: 15 + mon osdmap full prune interval: 2 + mon osdmap full prune txsize: 2 +roles: +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] diff --git a/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro b/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro new file mode 120000 index 00000000000..a5b729b9efa --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/crimson-supported-all-distro @@ -0,0 +1 @@ +.qa/distros/crimson-supported-all-distro/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml b/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml new file mode 120000 index 00000000000..2bf67af1b18 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/crimson_qa_overrides.yaml @@ -0,0 +1 @@ +.qa/config/crimson_qa_overrides.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/deploy/.qa b/qa/suites/crimson-rados/osd_shards/deploy/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml b/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml new file mode 100644 index 00000000000..fd1c091541e --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/ceph.yaml @@ -0,0 +1,2 @@ +tasks: +- install: \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled b/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled new file mode 100644 index 00000000000..0c2062240ee --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/deploy/cephadm.yaml.disabled @@ -0,0 +1,16 @@ +# no need to verify os + flavor + sha1 +verify_ceph_hash: false +tasks: +- cephadm: + conf: + mgr: + debug ms: 1 + debug mgr: 20 + debug osd: 10 +- cephadm.shell: + mon.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/.qa b/qa/suites/crimson-rados/osd_shards/objectstore/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore-rbm.yaml.disabled b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore-rbm.yaml.disabled new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml new file mode 120000 index 00000000000..46903a8c942 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/objectstore/seastore/seastore_segmented.yaml @@ -0,0 +1 @@ +.qa/objectstore_crimson/seastore/segmented$/crimson_seastore_segmented.yaml \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/tasks/.qa b/qa/suites/crimson-rados/osd_shards/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml b/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml new file mode 100644 index 00000000000..2c4239a9a30 --- /dev/null +++ b/qa/suites/crimson-rados/osd_shards/tasks/crimson_fio_restart.yaml @@ -0,0 +1,123 @@ +overrides: + ceph: + log-ignorelist: + - PG_DEGRADED + - OSD_DOWN + - OSD_HOST_DOWN + - OSD_ROOT_DOWN + - PG_AVAILABILITY + osd-mkfs-args: + - --crimson_cpu_num + - "3" +tasks: +- ceph: + pre-mgr-commands: + - echo PRE_MGR_COMMAND_RUNNING + - sudo ceph -s + - sudo ceph config set osd crimson_cpu_num 3 +- exec: + mon.a: + - ceph -s + +- exec: + client.0: + - | + set -ex + if ! ceph osd pool ls | grep -qx 'testpool'; then + ceph osd pool create testpool 64 64 + fi + ceph osd pool application enable testpool rbd || true + ceph -s + if ! rbd info testpool/testimage; then + rbd create testpool/testimage --size 10G --image-format=2 + fi +- exec: + client.0: + - | + set -ex + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards + ceph config set osd seastore_require_partition_count_match_reactor_count false + ceph config set osd.0 crimson_cpu_num 2 + ceph config set osd.1 crimson_cpu_num 2 + ceph config set osd.2 crimson_cpu_num 2 + +- ceph.restart: + daemons: [osd.0, osd.1, osd.2] + wait-for-healthy: false + wait-for-osds-up: true + +- exec: + osd.0: + - | + set -ex + ceph health detail + ceph daemon osd.0 config show +- exec: + client.0: + - | + set -ex + rbd info testpool/testimage + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards + ceph config set osd.0 crimson_cpu_num 5 + ceph config set osd.1 crimson_cpu_num 5 + ceph config set osd.2 crimson_cpu_num 5 +- ceph.restart: + daemons: [osd.0, osd.1, osd.2] + wait-for-healthy: false + wait-for-osds-up: true + +- exec: + osd.0: + - | + set -ex + ceph health detail + ceph daemon osd.0 config show + +- exec: + client.0: + - | + set -ex + rbd info testpool/testimage + fio --name=rbdtest \ + --ioengine=rbd \ + --pool=testpool \ + --rbdname=testimage \ + --clientname=admin \ + --rw=randwrite \ + --bs=4k \ + --iodepth=32 \ + --numjobs=4 \ + --runtime=120 \ + --time_based \ + --group_reporting \ + --direct=1 \ + --invalidate=0 + ceph daemon osd.0 dump_store_shards \ No newline at end of file diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index b29448b77c2..aaa89187923 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -1029,6 +1029,10 @@ def cluster(ctx, config): ctx.disk_config.remote_to_roles_to_dev_fstype[remote][role] = fs devs_to_clean[remote].append(mnt_point) + overrides = ctx.config.get('overrides', {}) + ceph_overrides = overrides.get('ceph', {}) + mkfs_args = ceph_overrides.get('osd-mkfs-args', []) + log.info("OSD mkfs args = %s", mkfs_args) for role in teuthology.cluster_roles_of_type(roles_for_host, 'osd', cluster_name): _, _, id_ = teuthology.split_role(role) try: @@ -1043,6 +1047,8 @@ def cluster(ctx, config): '--mkkey', '-i', id_, '--monmap', monmap_path] + if mkfs_args: + args.extend(mkfs_args) log_path = f'/var/log/ceph/{cluster_name}-osd.{id_}.log' create_log_cmd, args = \ maybe_redirect_stderr(config, 'osd', args, log_path)