From: Adam King <adking@redhat.com>
Date: Mon, 15 Jul 2024 19:19:22 +0000 (-0400)
Subject: qa/upgrade: use staggered upgrade features for reef-x/stress-split
X-Git-Tag: v19.2.1~73^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e377b5a11b5f7a8a91128bfbb3f2cdf330a0216d;p=ceph.git

qa/upgrade: use staggered upgrade features for reef-x/stress-split

This test was trying to partially upgrade the mons and OSDs by
kicking off an upgrade and then checking every 2 seconds if
enough had been upgraded. Since staggered upgrade parameters
were present in the initial reef release (not true for quincy)
it makes sense to use them instead in order to do this in a
more controlled manner.

Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit f1ca0c79de1580038226c03ff0c4d24f93022c59)
---

diff --git a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
index 00822228f4a13..4cd05432d5fc6 100644
--- a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
@@ -53,37 +53,33 @@ first-half-sequence:
       - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60
       - ceph config set global log_to_journald false --force
 
-      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-      - ceph orch ps
+      - echo wait for mgr daemons to upgrade
+      # upgrade the mgr daemons first
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
-      - ceph orch ps
-      - ceph orch upgrade pause
+      # upgrade 1 of 3 mon daemons, then wait 60 seconds
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
       - sleep 60
-      - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
-      - ceph orch ps
-      - ceph orch upgrade pause
+      # upgrade one more mon daemon (to get us to 2/3 upgraded) and wait 60 seconds
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
       - sleep 60
-      - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done"
-      - ceph orch ps
-      - ceph orch upgrade pause
+      # upgrade final mon daemon and wait 60 seconds
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
       - sleep 60
-      - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
-      - ceph orch upgrade pause
-      - ceph orch ps
-
-      - ceph orch ps
-      - ceph versions
+      # upgrade 4 of the 8 OSDs
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
 
 #################
@@ -108,10 +104,11 @@ second-half-sequence:
     - cephadm.shell:
         env: [sha1]
         mon.a:
-          - ceph orch upgrade resume
           - sleep 60
 
           - echo wait for upgrade to complete
+          # upgrade whatever is left
+          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
           - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
           - echo upgrade complete