qa/upgrade: fix checks to make sure upgrade is still in progress

Without checking both for the upgrade being in progress and that
the status isn't reporting an error, we can end up in a scenario
where the test is just waiting for an upgrade that has already
been marked failed and will never complete. This same sort of
change was already done in the orch suite upgrade tests and
has helped with jobs timing out there

Fixes: https://tracker.ceph.com/issues/65546
Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 61a48c5cedd7a6120e466ac3411b0165af62e878)

diff --git a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
index a3f0888da1d80a173b1c23a8bb0516c687bc6a54..3aec52ce126437e552e46fe026cc70af149a31ff 100644 (file)
--- a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
+++ b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
index b9bca65fb6f850e72db61533f0e792dbe16d4e71..378c22e3e35869f467a6b2656d0ecfea90e44680 100644 (file)
--- a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
@@ -58,28 +58,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 

diff --git a/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml
index a3f0888da1d80a173b1c23a8bb0516c687bc6a54..3aec52ce126437e552e46fe026cc70af149a31ff 100644 (file)
--- a/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml
+++ b/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml
@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
index 1add459f439e1843276b9854d7a28c44eb43a154..00822228f4a133d3deee04b0ba2393bc814f5b9b 100644 (file)
--- a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
@@ -57,28 +57,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 
@@ -112,7 +112,7 @@ second-half-sequence:
           - sleep 60
 
           - echo wait for upgrade to complete
-          - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+          - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
           - echo upgrade complete
           - ceph orch ps

diff --git a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
index bfccab47f584a10dcf07175dd6159c3af4292cfb..cefaee6963928632c2a5054bd4a6977b2452d3dc 100644 (file)
--- a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
+++ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
@@ -56,7 +56,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml
index a6412f229c4d71054a52ef74a7d606a8b7a3297c..ad4f990afb15b0b9f06ea5016cd9e06ee297c406 100644 (file)
--- a/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml
+++ b/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml
@@ -55,7 +55,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'