From eb2dfacee2bec0acdeac4ee53923ae5d5527a54b Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 15 Jul 2024 15:02:27 -0400 Subject: [PATCH] qa/upgrade: fix checks to make sure upgrade is still in progress Without checking both for the upgrade being in progress and that the status isn't reporting an error, we can end up in a scenario where the test is just waiting for an upgrade that has already been marked failed and will never complete. This same sort of change was already done in the orch suite upgrade tests and has helped with jobs timing out there Fixes: https://tracker.ceph.com/issues/65546 Signed-off-by: Adam King (cherry picked from commit 61a48c5cedd7a6120e466ac3411b0165af62e878) --- .../upgrade/pacific-x/parallel/upgrade-sequence.yaml | 2 +- qa/suites/upgrade/pacific-x/stress-split/1-start.yaml | 10 +++++----- .../upgrade/quincy-x/parallel/upgrade-sequence.yaml | 2 +- qa/suites/upgrade/quincy-x/stress-split/1-start.yaml | 8 ++++---- .../upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml | 2 +- .../upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml index a3f0888da1d..3aec52ce126 100644 --- a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml +++ b/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml @@ -7,7 +7,7 @@ upgrade-sequence: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml b/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml index 9a552df9991..13c8a2fa0c5 100644 --- a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml +++ b/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml @@ -58,28 +58,28 @@ first-half-sequence: - ceph orch ps - echo wait for minority of mons to upgrade - - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done + - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done" + - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch upgrade pause - ceph orch ps @@ -113,7 +113,7 @@ second-half-sequence: - sleep 60 - echo wait for upgrade to complete - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps diff --git a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml index a3f0888da1d..3aec52ce126 100644 --- a/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml +++ b/qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml @@ -7,7 +7,7 @@ upgrade-sequence: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml index b9bca65fb6f..378c22e3e35 100644 --- a/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml +++ b/qa/suites/upgrade/quincy-x/stress-split/1-start.yaml @@ -58,28 +58,28 @@ first-half-sequence: - ceph orch ps - echo wait for minority of mons to upgrade - - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done + - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done" + - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done" + - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done" - ceph orch upgrade pause - ceph orch ps diff --git a/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml index 28b9eb29fc9..3e18e509775 100644 --- a/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml +++ b/qa/suites/upgrade/telemetry-upgrade/pacific-x/1-tasks.yaml @@ -38,7 +38,7 @@ tasks: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' diff --git a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml index cd6609a6d41..1d5bb080c78 100644 --- a/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml +++ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml @@ -37,7 +37,7 @@ tasks: mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done + - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' -- 2.39.5