From cf205b1b23d4eb7b466861180ba1cbf4030f0dc1 Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 15 Jul 2024 15:02:27 -0400 Subject: [PATCH] qa/upgrade: fix checks to make sure upgrade is still in progress Without checking both for the upgrade being in progress and that the status isn't reporting an error, we can end up in a scenario where the test is just waiting for an upgrade that has already been marked failed and will never complete. This same sort of change was already done in the orch suite upgrade tests and has helped with jobs timing out there Fixes: https://tracker.ceph.com/issues/65546 Signed-off-by: Adam King (cherry picked from commit 61a48c5cedd7a6120e466ac3411b0165af62e878) --- .../upgrade/quincy-x/parallel/upgrade-sequence.yaml | 2 +- qa/suites/upgrade/quincy-x/stress-split/1-start.yaml | 8 ++++---- .../upgrade/reef-x/parallel/upgrade-sequence.yaml | 2 +- qa/suites/upgrade/reef-x/stress-split/1-start.yaml | 10 +++++----- .../upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml | 2 +- .../upgrade/telemetry-upgrade/reef-x/1-tasks.yaml | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml index a3f0888da1d80..3aec52ce12643 100644 --- a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml +++ b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml @@ -7,7 +7,7 @@ upgrade-sequence: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml index b9bca65fb6f85..378c22e3e3586 100644 --- a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml +++ b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml @@ -58,28 +58,28 @@ first-half-sequence: - ceph orch ps - echo wait for minority of mons to upgrade - - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done + - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done" + - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch upgrade pause - ceph orch ps diff --git a/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml index a3f0888da1d80..3aec52ce12643 100644 --- a/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml +++ b/qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml @@ -7,7 +7,7 @@ upgrade-sequence: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml index 1add459f439e1..00822228f4a13 100644 --- a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml +++ b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml @@ -57,28 +57,28 @@ first-half-sequence: - ceph orch ps - echo wait for minority of mons to upgrade - - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done + - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done" + - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch upgrade pause - ceph orch ps @@ -112,7 +112,7 @@ second-half-sequence: - sleep 60 - echo wait for upgrade to complete - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps diff --git a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml index bfccab47f584a..cefaee6963928 100644 --- a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml +++ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml @@ -56,7 +56,7 @@ tasks: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml index a6412f229c4d7..ad4f990afb15b 100644 --- a/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml +++ b/qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml @@ -55,7 +55,7 @@ tasks: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' -- 2.39.5