From: Adam King Date: Thu, 6 Oct 2022 16:02:01 +0000 (-0400) Subject: qa/cephadm: update staggered upgrade test to use --image when redeploying mgr X-Git-Tag: v18.1.0~729^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=10e89a53e90ad622546c5163e6a671d43611d23e;p=ceph.git qa/cephadm: update staggered upgrade test to use --image when redeploying mgr since we now start from pacific rather than octopus, this flag is supported Signed-off-by: Adam King --- diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml index b792c2857065..5f9677d68433 100644 --- a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml +++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml @@ -14,8 +14,6 @@ tasks: cephadm_git_url: https://github.com/ceph/ceph # avoid --cap-add=PTRACE + --privileged for older cephadm versions allow_ptrace: false - # deploy additional mons the "old" (octopus) way - add_mons_via_daemon_add: true avoid_pacific_features: true roles: diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml index 0ac9de158280..475cdaa5d28a 100644 --- a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml +++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml @@ -8,8 +8,6 @@ tasks: cephadm_git_url: https://github.com/ceph/ceph # avoid --cap-add=PTRACE + --privileged for older cephadm versions allow_ptrace: false - # deploy additional mons the "old" (octopus) way - add_mons_via_daemon_add: true avoid_pacific_features: true roles: diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml index bc318adf4832..111e0274a1c5 100644 --- a/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml +++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml @@ -6,7 +6,7 @@ tasks: - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - - ceph orch apply rgw r z --placement=2 --port=8000 + - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - sleep 120 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml index ebd06c92d262..109ccb9dbecd 100644 --- a/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml +++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml @@ -6,7 +6,7 @@ tasks: - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - - ceph orch apply rgw r z --placement=2 --port=8000 + - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force @@ -18,26 +18,26 @@ tasks: - ceph orch ls # doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code # until there is a stable version that contains it, we can test by manually upgrading a mgr daemon - - ceph config set mgr container_image quay.ceph.io/ceph-ci/ceph:$sha1 - - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" + - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 # gather more possible debugging info - ceph orch ps - ceph versions - ceph -s + - ceph health detail # check that there are two different versions found for mgr daemon (which implies we upgraded one) - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 # now try upgrading the other mgr - # we should now have access to --image flag for the daemon redeploy command - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 # gather more possible debugging info - ceph orch ps - ceph versions + - ceph health detail - ceph -s - ceph mgr fail - sleep 180 @@ -45,12 +45,14 @@ tasks: - ceph orch ps - ceph versions - ceph -s + - ceph health detail # now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 # debugging info - ceph orch ps + - ceph orch ls - ceph versions # to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on new new version # also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized @@ -63,12 +65,16 @@ tasks: - ceph versions | jq -e '.overall | length == 2' # check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons) - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' + - ceph orch upgrade status + - ceph health detail # upgrade only the mons on one of the two hosts - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - # verify tow different version seen for mons + # verify two different version seen for mons - ceph versions | jq -e '.mon | length == 2' + - ceph orch upgrade status + - ceph health detail # upgrade mons on the other hosts - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done @@ -78,6 +84,8 @@ tasks: - ceph versions | jq -e '.mon | keys' | grep $sha1 # verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons) - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' + - ceph orch upgrade status + - ceph health detail # upgrade exactly 2 osd daemons - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done @@ -86,6 +94,8 @@ tasks: - ceph versions | jq -e '.osd | length == 2' # verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds) - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' + - ceph orch upgrade status + - ceph health detail # upgrade one more osd - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done @@ -94,18 +104,24 @@ tasks: # verify now 8 daemons have been upgraded - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' # upgrade the rest of the osds + - ceph orch upgrade status + - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps # verify all osds are now on same version and version hash matches what we are upgrading to - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 + - ceph orch upgrade status + - ceph health detail # upgrade the rgw daemons using --services - - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.r.z + - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps # verify all rgw daemons on same version and version hash matches what we are upgrading to - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 + - ceph orch upgrade status + - ceph health detail # run upgrade one more time with no filter parameters to make sure anything left gets upgraded - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1