qa/upgrade: fix checks to make sure upgrade is still in progress

Without checking both for the upgrade being in progress and that
the status isn't reporting an error, we can end up in a scenario
where the test is just waiting for an upgrade that has already
been marked failed and will never complete. This same sort of
change was already done in the orch suite upgrade tests and
has helped with jobs timing out there

Fixes: https://tracker.ceph.com/issues/65546
Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 61a48c5cedd7a6120e466ac3411b0165af62e878)

diff --git a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
index a3f0888da1d80a173b1c23a8bb0516c687bc6a54..3aec52ce126437e552e46fe026cc70af149a31ff 100644 (file)
--- a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
+++ b/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml b/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
index 9a552df9991fd652100c1b68cd1fb71417490da1..13c8a2fa0c5a0f7646e2744a9f3eb3bda41f0506 100644 (file)
--- a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
@@ -58,28 +58,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 
@@ -113,7 +113,7 @@ second-half-sequence:
           - sleep 60
 
           - echo wait for upgrade to complete
-          - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+          - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
           - echo upgrade complete
           - ceph orch ps

diff --git a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
index a3f0888da1d80a173b1c23a8bb0516c687bc6a54..3aec52ce126437e552e46fe026cc70af149a31ff 100644 (file)
--- a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
+++ b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml
@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
index b9bca65fb6f850e72db61533f0e792dbe16d4e71..378c22e3e35869f467a6b2656d0ecfea90e44680 100644 (file)
--- a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml
@@ -58,28 +58,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 

diff --git a/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml
index 28b9eb29fc95afc398c40ee45cb341e70518e0a0..3e18e5097752c3ca095880cf3b32c0ef95ff5632 100644 (file)
--- a/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml
+++ b/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml
@@ -38,7 +38,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'

diff --git a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
index cd6609a6d41ef4e7de5d93e0f305f80bceb8fba5..1d5bb080c78f1d79183b404621eedf8943a6060f 100644 (file)
--- a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
+++ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml
@@ -37,7 +37,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'