qa/suites/orch/cephadm: stop upgrade tests if failures are seen

Otherwise the tests may run forever. This was already done for
mds upgrade sequence, justadding it in the other two places here

Related to: https://tracker.ceph.com/issues/53939

Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 017aa9cfe8362e8512a581e39850ce70bd1ce82f)

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
index 6d10b3576e2ed851ab538077931e992dca9baca6..362e0373416ea1478aaed9aa8c8fb1b47e273393 100644 (file)
--- a/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
@@ -16,7 +16,7 @@ upgrade-tasks:
   - cephadm.shell:
       env: [sha1]
       host.a:
-        - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+        - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
         - ceph orch ps
         - ceph versions
         - echo "wait for servicemap items w/ changing names to refresh"

diff --git a/qa/suites/orch/cephadm/upgrade/4-wait.yaml b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
index dfac1847afdc27cdd196f4a7a5de9ba8a59ad42b..58afe00c5be88782d17b36ffc39617a93b58cce3 100644 (file)
--- a/qa/suites/orch/cephadm/upgrade/4-wait.yaml
+++ b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
@@ -2,7 +2,7 @@ tasks:
 - cephadm.shell:
     env: [sha1]
     mon.a:
-      - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
       - ceph orch ps
       - ceph versions
       - echo "wait for servicemap items w/ changing names to refresh"