From 0bdfb93aa34dcfac55781f70e24edac3f1090bbe Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Thu, 29 May 2025 12:02:38 -0400 Subject: [PATCH] qa: add upgrade suite for cephx To test upgrade paths for "aes" key type to "aes256k" including the expected flows for service key updates and entity rotation. Signed-off-by: Patrick Donnelly --- qa/suites/rados/upgrade/cephx/.qa | 1 + qa/suites/rados/upgrade/cephx/X/% | 0 qa/suites/rados/upgrade/cephx/X/.qa | 1 + .../rados/upgrade/cephx/X/centos_latest.yaml | 1 + qa/suites/rados/upgrade/cephx/X/conf.yaml | 13 +++++++ .../rados/upgrade/cephx/X/ignore_health.yaml | 25 ++++++++++++ .../rados/upgrade/cephx/X/ms_bind/v1.yaml | 17 +++++++++ .../rados/upgrade/cephx/X/ms_bind/v2.yaml | 17 +++++++++ qa/suites/rados/upgrade/cephx/X/roles.yaml | 4 ++ qa/suites/rados/upgrade/cephx/X/tasks/% | 0 qa/suites/rados/upgrade/cephx/X/tasks/.qa | 1 + .../upgrade/cephx/X/tasks/0-install.yaml | 2 + .../rados/upgrade/cephx/X/tasks/1-ceph.yaml | 10 +++++ .../rados/upgrade/cephx/X/tasks/2-debug.yaml | 13 +++++++ .../upgrade/cephx/X/tasks/3-workload/.qa | 1 + .../cephx/X/tasks/3-workload/radosbench.yaml | 11 ++++++ .../X/tasks/4-cephx-service-upgrade.yaml | 14 +++++++ .../cephx/X/tasks/5-wipe-service-keys/.qa | 1 + .../cephx/X/tasks/5-wipe-service-keys/no.yaml | 3 ++ .../X/tasks/5-wipe-service-keys/yes.yaml | 6 +++ .../cephx/X/tasks/6-daemon-restart/.qa | 1 + .../cephx/X/tasks/6-daemon-restart/no.yaml | 0 .../cephx/X/tasks/6-daemon-restart/sleep.yaml | 4 ++ .../cephx/X/tasks/6-daemon-restart/yes.yaml | 25 ++++++++++++ .../cephx/X/tasks/7-workload-after-rotate/.qa | 1 + .../7-workload-after-rotate/radosbench.yaml | 6 +++ qa/suites/rados/upgrade/cephx/release-X/% | 0 qa/suites/rados/upgrade/cephx/release-X/.qa | 1 + .../cephx/release-X/centos_latest.yaml | 1 + .../rados/upgrade/cephx/release-X/conf.yaml | 13 +++++++ .../cephx/release-X/ignore_health.yaml | 25 ++++++++++++ .../upgrade/cephx/release-X/ms_bind/v1.yaml | 17 +++++++++ .../upgrade/cephx/release-X/ms_bind/v2.yaml | 17 +++++++++ .../rados/upgrade/cephx/release-X/roles.yaml | 5 +++ .../rados/upgrade/cephx/release-X/tasks/% | 0 .../rados/upgrade/cephx/release-X/tasks/.qa | 1 + .../upgrade/cephx/release-X/tasks/0-from/.qa | 1 + .../cephx/release-X/tasks/0-from/v18.2.7.yaml | 38 +++++++++++++++++++ .../cephx/release-X/tasks/0-from/v19.2.2.yaml | 38 +++++++++++++++++++ .../cephx/release-X/tasks/1-debug.yaml | 12 ++++++ .../cephx/release-X/tasks/2-workload/.qa | 1 + .../tasks/2-workload/radosbench.yaml | 23 +++++++++++ .../cephx/release-X/tasks/3-upgrade.yaml | 32 ++++++++++++++++ .../tasks/4-cephx-service-upgrade.yaml | 14 +++++++ .../release-X/tasks/5-wipe-service-keys/.qa | 1 + .../tasks/5-wipe-service-keys/no.yaml | 3 ++ .../tasks/5-wipe-service-keys/yes.yaml | 12 ++++++ .../release-X/tasks/6-daemon-restart/.qa | 1 + .../release-X/tasks/6-daemon-restart/no.yaml | 0 .../release-X/tasks/6-daemon-restart/yes.yaml | 25 ++++++++++++ .../tasks/7-workload-after-rotate/.qa | 1 + .../7-workload-after-rotate/radosbench.yaml | 6 +++ 52 files changed, 466 insertions(+) create mode 120000 qa/suites/rados/upgrade/cephx/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/% create mode 120000 qa/suites/rados/upgrade/cephx/X/.qa create mode 120000 qa/suites/rados/upgrade/cephx/X/centos_latest.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/conf.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/ignore_health.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/roles.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/% create mode 120000 qa/suites/rados/upgrade/cephx/X/tasks/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml create mode 120000 qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml create mode 120000 qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml create mode 120000 qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/no.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml create mode 120000 qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa create mode 100644 qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/% create mode 120000 qa/suites/rados/upgrade/cephx/release-X/.qa create mode 120000 qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/conf.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/roles.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/% create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/.qa create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/no.yaml create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml create mode 120000 qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa create mode 100644 qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml diff --git a/qa/suites/rados/upgrade/cephx/.qa b/qa/suites/rados/upgrade/cephx/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/% b/qa/suites/rados/upgrade/cephx/X/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/X/.qa b/qa/suites/rados/upgrade/cephx/X/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml b/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml new file mode 120000 index 00000000000..bd9854e7029 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/conf.yaml b/qa/suites/rados/upgrade/cephx/X/conf.yaml new file mode 100644 index 00000000000..cdae1643007 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/conf.yaml @@ -0,0 +1,13 @@ +overrides: + ceph: + conf: + global: + debug ms: 1 + debug auth: 30 + debug monc: 30 + log to file: true + auth mon ticket ttl: 300 + auth service ticket ttl: 120 + mon: + debug mon: 30 + debug ms: 5 diff --git a/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml b/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml new file mode 100644 index 00000000000..e96acf51ece --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/ignore_health.yaml @@ -0,0 +1,25 @@ +overrides: + ceph: + log-ignorelist: + - do not have an application enabled + - application not enabled + - or freeform for custom applications + - POOL_APP_NOT_ENABLED + - is down + - OSD_DOWN + - mons down + - mon down + - MON_DOWN + - out of quorum + - PG_AVAILABILITY + - PG_DEGRADED + - Reduced data availability + - Degraded data redundancy + - pg .* is stuck inactive + - pg .* is .*degraded + - FS_DEGRADED + - OSDMAP_FLAGS + - OSD_UPGRADE_FINISHED + - OSD_HOST_DOWN + - filesystem is degraded + - evicting unresponsive client diff --git a/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml b/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml new file mode 100644 index 00000000000..439563fd06e --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/ms_bind/v1.yaml @@ -0,0 +1,17 @@ +overrides: + ceph: + conf: + global: + mon warn on msgr2 not enabled: false + osd: + ms bind msgr1: true + ms bind msgr2: false + mds: + ms bind msgr1: true + ms bind msgr2: false + mon: + ms bind msgr1: true + ms bind msgr2: false + mgr: + ms bind msgr1: true + ms bind msgr2: false diff --git a/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml b/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml new file mode 100644 index 00000000000..4e81054daf3 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/ms_bind/v2.yaml @@ -0,0 +1,17 @@ +overrides: + ceph: + conf: + global: + mon warn on msgr2 not enabled: false + osd: + ms bind msgr1: false + ms bind msgr2: true + mds: + ms bind msgr1: false + ms bind msgr2: true + mon: + ms bind msgr1: false + ms bind msgr2: true + mgr: + ms bind msgr1: false + ms bind msgr2: true diff --git a/qa/suites/rados/upgrade/cephx/X/roles.yaml b/qa/suites/rados/upgrade/cephx/X/roles.yaml new file mode 100644 index 00000000000..4e98cf0b6d3 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/roles.yaml @@ -0,0 +1,4 @@ +roles: +- [mon.a, mds.a, mgr.x, osd.0, osd.1] +- [mon.b, mon.c, mds.b, mgr.y, osd.2, osd.3] +- [client.0] diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/% b/qa/suites/rados/upgrade/cephx/X/tasks/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml new file mode 100644 index 00000000000..e6f464416cc --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/0-install.yaml @@ -0,0 +1,2 @@ +tasks: +- install: diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml new file mode 100644 index 00000000000..eefb7a26c4e --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/1-ceph.yaml @@ -0,0 +1,10 @@ +tasks: +- ceph: + cluster-conf: + global: + auth service cipher: aes + cephx preferred cipher: aes + mon: + mon auth allow insecure key: true + cephx: + key_type: aes diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml new file mode 100644 index 00000000000..f0352e61805 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/2-debug.yaml @@ -0,0 +1,13 @@ +tasks: +- full_sequential_finally: + - exec: + mon.a: + - sleep 1m +- exec: + mon.a: + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.a config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.b config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.c config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config dump + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml new file mode 100644 index 00000000000..cc18fab3789 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/3-workload/radosbench.yaml @@ -0,0 +1,11 @@ +radosbench: + sequential_yield: + - radosbench: + extra_args: --log-to-stderr=false --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0 + clients: [client.0] + time: 300 + unique_pool: true + +tasks: + - sequential_yield: + - radosbench diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml new file mode 100644 index 00000000000..c193a55f628 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/4-cephx-service-upgrade.yaml @@ -0,0 +1,14 @@ +tasks: +- ceph.key_rotate: + daemons: [mon.*] + key_type: aes256k +- ceph.key_rotate: + daemons: [mgr.*, osd.*, mds.*] + key_type: aes256k +- exec: + mon.a: + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json-pretty dump-keys + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq 'any(.data.secrets[] | select(.key.type == 1 or .key.type == 2 or .key.type == 4 or .key.type == 16); .val.key.type != 2)' + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq '.data.rotating_secrets | all( .val.secrets | all(.val.key.type == 1) )' + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config set global auth_service_cipher aes256k diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml new file mode 100644 index 00000000000..2290d553550 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/no.yaml @@ -0,0 +1,3 @@ +teuthology: + variables: + wipe_service_keys: false diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml new file mode 100644 index 00000000000..b7660744d10 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/5-wipe-service-keys/yes.yaml @@ -0,0 +1,6 @@ +tasks: +- exec: + mon.a: + - ceph auth wipe-rotating-service-keys + - ceph auth --format=json-pretty dump-keys + - ceph auth --format=json dump-keys | jq '.data.rotating_secrets | all( if .key == 32 then (.val.secrets | all(.val.key.type == 1)) else (.val.secrets | all(.val.key.type == 2)) end )' diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/no.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/no.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml new file mode 100644 index 00000000000..398439f1ced --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/sleep.yaml @@ -0,0 +1,4 @@ +# Sleep for ticket refresh. +tasks: +- sleep: + duration: 500 diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml new file mode 100644 index 00000000000..56c72716489 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/6-daemon-restart/yes.yaml @@ -0,0 +1,25 @@ +overrides: + ceph: + log-ignorelist: + - OSD_DOWN + - OSD_ROOT_DOWN +tasks: +- sleep: + duration: 10 +- ceph.restart: + daemons: [mon.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [mgr.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [osd.*] + mon-health-to-clog: false + wait-for-healthy: true + wait-for-osds-up: true +- ceph.restart: + daemons: [mds.*] + mon-health-to-clog: false + wait-for-healthy: true diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml new file mode 100644 index 00000000000..55bd3780f5f --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/7-workload-after-rotate/radosbench.yaml @@ -0,0 +1,6 @@ +tasks: + - radosbench: + extra_args: --log-to-stderr=false --log-to-file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0 + clients: [client.0] + time: 10 + unique_pool: true diff --git a/qa/suites/rados/upgrade/cephx/release-X/% b/qa/suites/rados/upgrade/cephx/release-X/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/release-X/.qa b/qa/suites/rados/upgrade/cephx/release-X/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml b/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml new file mode 120000 index 00000000000..bd9854e7029 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/conf.yaml b/qa/suites/rados/upgrade/cephx/release-X/conf.yaml new file mode 100644 index 00000000000..cdae1643007 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/conf.yaml @@ -0,0 +1,13 @@ +overrides: + ceph: + conf: + global: + debug ms: 1 + debug auth: 30 + debug monc: 30 + log to file: true + auth mon ticket ttl: 300 + auth service ticket ttl: 120 + mon: + debug mon: 30 + debug ms: 5 diff --git a/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml b/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml new file mode 100644 index 00000000000..e96acf51ece --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/ignore_health.yaml @@ -0,0 +1,25 @@ +overrides: + ceph: + log-ignorelist: + - do not have an application enabled + - application not enabled + - or freeform for custom applications + - POOL_APP_NOT_ENABLED + - is down + - OSD_DOWN + - mons down + - mon down + - MON_DOWN + - out of quorum + - PG_AVAILABILITY + - PG_DEGRADED + - Reduced data availability + - Degraded data redundancy + - pg .* is stuck inactive + - pg .* is .*degraded + - FS_DEGRADED + - OSDMAP_FLAGS + - OSD_UPGRADE_FINISHED + - OSD_HOST_DOWN + - filesystem is degraded + - evicting unresponsive client diff --git a/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml new file mode 100644 index 00000000000..439563fd06e --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v1.yaml @@ -0,0 +1,17 @@ +overrides: + ceph: + conf: + global: + mon warn on msgr2 not enabled: false + osd: + ms bind msgr1: true + ms bind msgr2: false + mds: + ms bind msgr1: true + ms bind msgr2: false + mon: + ms bind msgr1: true + ms bind msgr2: false + mgr: + ms bind msgr1: true + ms bind msgr2: false diff --git a/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml new file mode 100644 index 00000000000..4e81054daf3 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/v2.yaml @@ -0,0 +1,17 @@ +overrides: + ceph: + conf: + global: + mon warn on msgr2 not enabled: false + osd: + ms bind msgr1: false + ms bind msgr2: true + mds: + ms bind msgr1: false + ms bind msgr2: true + mon: + ms bind msgr1: false + ms bind msgr2: true + mgr: + ms bind msgr1: false + ms bind msgr2: true diff --git a/qa/suites/rados/upgrade/cephx/release-X/roles.yaml b/qa/suites/rados/upgrade/cephx/release-X/roles.yaml new file mode 100644 index 00000000000..5baa2657abc --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/roles.yaml @@ -0,0 +1,5 @@ +roles: +- [mon.a, mds.a, mgr.x, osd.0, osd.1] +- [mon.b, mon.c, mds.b, mgr.y, osd.2, osd.3] +- [client.0] +- [client.1] diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/% b/qa/suites/rados/upgrade/cephx/release-X/tasks/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml new file mode 100644 index 00000000000..dac4fb18e7f --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v18.2.7.yaml @@ -0,0 +1,38 @@ +meta: +- desc: | + install ceph/reef v18.2.7 +tasks: +- install: + tag: v18.2.7 + exclude_packages: + - librados3 + - ceph-mgr-dashboard + - ceph-mgr-diskprediction-local + - ceph-mgr-rook + - ceph-mgr-cephadm + - cephadm + - ceph-volume + extra_packages: ['librados2'] +- print: "**** done installing v18.2.7" +- ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_ + - \(MDS_ + - \(OSD_ + - \(MON_DOWN\) + - \(CACHE_POOL_ + - \(POOL_ + - \(MGR_DOWN\) + - \(PG_ + - \(SMALLER_PGP_NUM\) + - Monitor daemon marked osd + - Behind on trimming + - Manager daemon + conf: + global: + mon warn on pool no app: false +- exec: + osd.0: + - ceph osd set-require-min-compat-client reef +- print: "**** done ceph" diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml new file mode 100644 index 00000000000..651f8d3ccde --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/0-from/v19.2.2.yaml @@ -0,0 +1,38 @@ +meta: +- desc: | + install ceph/squid v19.2.2 +tasks: +- install: + tag: v19.2.2 + exclude_packages: + - librados3 + - ceph-mgr-dashboard + - ceph-mgr-diskprediction-local + - ceph-mgr-rook + - ceph-mgr-cephadm + - cephadm + - ceph-volume + extra_packages: ['librados2'] +- print: "**** done installing squid v19.2.2" +- ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_ + - \(MDS_ + - \(OSD_ + - \(MON_DOWN\) + - \(CACHE_POOL_ + - \(POOL_ + - \(MGR_DOWN\) + - \(PG_ + - \(SMALLER_PGP_NUM\) + - Monitor daemon marked osd + - Behind on trimming + - Manager daemon + conf: + global: + mon warn on pool no app: false +- exec: + osd.0: + - ceph osd set-require-min-compat-client squid +- print: "**** done ceph" diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml new file mode 100644 index 00000000000..233edca93e3 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/1-debug.yaml @@ -0,0 +1,12 @@ +tasks: +- full_sequential_finally: + - exec: + mon.a: + - sleep 1m +- exec: + mon.a: + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.a config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.b config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 tell mon.c config diff + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 config dump + - ceph --log-to-stderr=true --log_to_file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml new file mode 100644 index 00000000000..4320ec624e4 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/2-workload/radosbench.yaml @@ -0,0 +1,23 @@ +# We can't set auth_exit_on_failure here because squid/reef do not understand that switch. +#teuthology: +# postmerge: +# - | +# if false and yaml.teuthology.variables.wipe_service_keys then +# yaml.radosbench.sequential_yield[0].radosbench.auth_exit_on_failure = 99 +# yaml.radosbench.sequential_yield[0].radosbench.expected_rc = 99 +# end + +# N.B. because `rados bench` has sessions open with the OSDs, we do not expect +# it to fail any auth after upgrade / rotation / session key wipe. It will only +# fail new connections with OSDs. +radosbench: + sequential_yield: + - radosbench: + extra_args: --log-to-stderr=false --log-to-file=true --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0 + clients: [client.0] + time: 300 + unique_pool: true + +tasks: + - sequential_yield: + - radosbench diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml new file mode 100644 index 00000000000..a327fd1d8cf --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/3-upgrade.yaml @@ -0,0 +1,32 @@ +tasks: +- install.upgrade: + mon.a: + mon.b: + client.0: +- ceph.restart: + daemons: [mgr.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [mon.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [osd.*] + mon-health-to-clog: false + wait-for-osds-up: true + wait-for-healthy: false +- exec: + mon.a: + - ceph versions + - ceph osd dump -f json-pretty + - ceph osd require-osd-release tentacle + - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done +- ceph.restart: + daemons: [mds.*] + mon-health-to-clog: false + wait-for-healthy: true +- exec: + mon.a: + - ceph versions + - ceph fs dump diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml new file mode 100644 index 00000000000..44f1ab2877b --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/4-cephx-service-upgrade.yaml @@ -0,0 +1,14 @@ +tasks: +- ceph.key_rotate: + daemons: [mon.*] + key_type: aes256k +- ceph.key_rotate: + daemons: [mgr.*, osd.*, mds.*] + key_type: aes256k +- exec: + mon.a: + - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth ls + - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json-pretty dump-keys + - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq 'any(.data.secrets[] | select(.key.type == 1 or .key.type == 2 or .key.type == 4 or .key.type == 16); .val.key.type != 2)' + - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 auth --format=json dump-keys | jq '.data.rotating_secrets | all( .val.secrets | all(.val.key.type == 1) )' + - ceph --debug_ms=5 --debug_auth=30 --debug_monc=30 config set global auth_service_cipher aes256k diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml new file mode 100644 index 00000000000..2290d553550 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/no.yaml @@ -0,0 +1,3 @@ +teuthology: + variables: + wipe_service_keys: false diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml new file mode 100644 index 00000000000..e4ee1a4ec58 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/5-wipe-service-keys/yes.yaml @@ -0,0 +1,12 @@ +# N.B. in order to effect a service key wipe, the service daemons must be +# restarted next. During this time, service daemons will be inaccessible to new +# clients. +teuthology: + variables: + wipe_service_keys: true +tasks: +- exec: + mon.a: + - ceph auth wipe-rotating-service-keys + - ceph auth --format=json-pretty dump-keys + - ceph auth --format=json dump-keys | jq '.data.rotating_secrets | all( if .key == 32 then (.val.secrets | all(.val.key.type == 1)) else (.val.secrets | all(.val.key.type == 2)) end )' diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/no.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/no.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml new file mode 100644 index 00000000000..56c72716489 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/6-daemon-restart/yes.yaml @@ -0,0 +1,25 @@ +overrides: + ceph: + log-ignorelist: + - OSD_DOWN + - OSD_ROOT_DOWN +tasks: +- sleep: + duration: 10 +- ceph.restart: + daemons: [mon.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [mgr.*] + mon-health-to-clog: false + wait-for-healthy: true +- ceph.restart: + daemons: [osd.*] + mon-health-to-clog: false + wait-for-healthy: true + wait-for-osds-up: true +- ceph.restart: + daemons: [mds.*] + mon-health-to-clog: false + wait-for-healthy: true diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml new file mode 100644 index 00000000000..1678f632d79 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/7-workload-after-rotate/radosbench.yaml @@ -0,0 +1,6 @@ +tasks: + - radosbench: + extra_args: --debug_ms=5 --debug_auth=30 --debug_monc=30 --mon_client_target_rank=0 + clients: [client.0] + time: 10 + unique_pool: true -- 2.39.5