From 6b4ac696fb2123ab2d8c8fc237158e178cf49ba4 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 1 Oct 2021 12:06:50 -0400 Subject: [PATCH] qa: add test for cephfs upgrade sequence This also checks max_mds>1 and allow_standby_replay are restored to previous values. Future work can add tests for multiple file systems (or volumes). Signed-off-by: Patrick Donnelly (cherry picked from commit b1420e5771927f5c659e0e5edbc5714035f3df09) --- qa/suites/fs/upgrade/mds_upgrade_sequence/% | 0 qa/suites/fs/upgrade/mds_upgrade_sequence/.qa | 1 + .../bluestore-bitmap.yaml | 1 + .../centos_8.3_container_tools_3.0.yaml | 1 + .../fs/upgrade/mds_upgrade_sequence/conf | 1 + .../upgrade/mds_upgrade_sequence/overrides/% | 0 .../mds_upgrade_sequence/overrides/.qa | 1 + .../overrides/pg-warn.yaml | 5 +++ .../overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../upgrade/mds_upgrade_sequence/roles.yaml | 11 +++++++ .../fs/upgrade/mds_upgrade_sequence/tasks/% | 0 .../fs/upgrade/mds_upgrade_sequence/tasks/.qa | 1 + .../mds_upgrade_sequence/tasks/0-v16.2.4.yaml | 25 ++++++++++++++ .../mds_upgrade_sequence/tasks/1-volume/% | 0 .../mds_upgrade_sequence/tasks/1-volume/.qa | 1 + .../tasks/1-volume/0-create.yaml | 5 +++ .../tasks/1-volume/1-ranks/.qa | 1 + .../tasks/1-volume/1-ranks/1.yaml | 4 +++ .../tasks/1-volume/1-ranks/2.yaml | 4 +++ .../tasks/1-volume/2-allow_standby_replay/.qa | 1 + .../1-volume/2-allow_standby_replay/no.yaml | 4 +++ .../1-volume/2-allow_standby_replay/yes.yaml | 4 +++ .../tasks/1-volume/3-verify.yaml | 7 ++++ .../mds_upgrade_sequence/tasks/2-client.yaml | 3 ++ .../tasks/3-upgrade-with-workload.yaml | 33 +++++++++++++++++++ .../mds_upgrade_sequence/tasks/4-verify.yaml | 5 +++ qa/suites/orch/cephadm/mds_upgrade_sequence | 1 + 28 files changed, 122 insertions(+) create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/.qa create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/conf create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml create mode 120000 qa/suites/orch/cephadm/mds_upgrade_sequence diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml new file mode 120000 index 0000000000000..fb603bc9a64c8 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/cephfs/objectstore-ec/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml new file mode 120000 index 0000000000000..479a5c26e451b --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.3_container_tools_3.0.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/conf b/qa/suites/fs/upgrade/mds_upgrade_sequence/conf new file mode 120000 index 0000000000000..6d47129847fad --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/conf @@ -0,0 +1 @@ +.qa/cephfs/conf/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml new file mode 100644 index 0000000000000..4ae54a40d3195 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + global: + mon pg warn min per osd: 0 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml new file mode 120000 index 0000000000000..74f39a49b27ed --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 0000000000000..b4528c0f8c096 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml new file mode 100644 index 0000000000000..bce4ecd34ccb3 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml @@ -0,0 +1,11 @@ +roles: +- - host.a + - client.0 + - osd.0 + - osd.1 + - osd.2 +- - host.b + - client.1 + - osd.3 + - osd.4 + - osd.5 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml new file mode 100644 index 0000000000000..e61d622669715 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml @@ -0,0 +1,25 @@ +meta: +- desc: | + setup ceph/pacific v16.2.4 + +tasks: +- install: + tag: v16.2.4 + exclude_packages: + - ceph-volume +- cephadm: + roleless: true + image: docker.io/ceph/ceph:v16.2.4 + cephadm_branch: v16.2.4 + cephadm_git_url: https://github.com/ceph/ceph + # needed for v16.2.4 due to --skip-admin-label + avoid_pacific_features: true +- print: "**** done starting v16.2.4" +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls + diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml new file mode 100644 index 0000000000000..5ee0022c6bd6e --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml @@ -0,0 +1,5 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs volume create cephfs --placement=4 + - ceph fs dump diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml new file mode 100644 index 0000000000000..8c1cd2fe0a1cd --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs max_mds 1 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml new file mode 100644 index 0000000000000..fcd3b1ea42ea7 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs max_mds 2 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml new file mode 100644 index 0000000000000..3dbc810899465 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs allow_standby_replay false diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml new file mode 100644 index 0000000000000..fb894425e990c --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs allow_standby_replay true diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml new file mode 100644 index 0000000000000..ec2a2a54f29de --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml @@ -0,0 +1,7 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs dump + - ceph --format=json fs dump | jq -e ".filesystems | length == 1" + - ceph --format=json mds versions | jq -e ". | add == 4" +- fs.pre_upgrade_save: diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml new file mode 100644 index 0000000000000..d7352c6bf5c2d --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml @@ -0,0 +1,3 @@ +tasks: +- kclient: +- print: "**** done kclient client" diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml new file mode 100644 index 0000000000000..200c4dcb58bb3 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml @@ -0,0 +1,33 @@ +tasks: +- parallel: + - upgrade-tasks + - workload-tasks + +upgrade-tasks: + sequential: + - cephadm.shell: + env: [sha1] + host.a: + - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force + - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force + - ceph config set global log_to_journald false --force + - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 + - cephadm.shell: + env: [sha1] + host.a: + - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; ceph fs dump; sleep 30 ; done + - ceph orch ps + - ceph versions + - echo "wait for servicemap items w/ changing names to refresh" + - sleep 60 + - ceph orch ps + - ceph versions + - ceph versions | jq -e '.overall | length == 1' + - ceph versions | jq -e '.overall | keys' | grep $sha1 + +workload-tasks: + sequential: + - workunit: + clients: + all: + - suites/fsstress.sh diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml new file mode 100644 index 0000000000000..c2b657e5a3b8a --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml @@ -0,0 +1,5 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs dump +- fs.post_upgrade_checks: diff --git a/qa/suites/orch/cephadm/mds_upgrade_sequence b/qa/suites/orch/cephadm/mds_upgrade_sequence new file mode 120000 index 0000000000000..24aa41c1039f9 --- /dev/null +++ b/qa/suites/orch/cephadm/mds_upgrade_sequence @@ -0,0 +1 @@ +.qa/suites/fs/upgrade/mds_upgrade_sequence/ \ No newline at end of file -- 2.39.5