test: Limit loops waiting for force-backfill/force-recovery to happen

Fixes: http://tracker.ceph.com/issues/38309
Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/qa/standalone/osd/osd-backfill-prio.sh b/qa/standalone/osd/osd-backfill-prio.sh
index 248ac6fb0c275136520bef57ed910c31be44eef6..6491cd19efd6086b82d65bd7a2cff95bd7a11c79 100755 (executable)
--- a/qa/standalone/osd/osd-backfill-prio.sh
+++ b/qa/standalone/osd/osd-backfill-prio.sh
@@ -48,6 +48,7 @@ function TEST_backfill_priority() {
     local OSDS=5
     # size 2 -> 1 means degraded by 1, so add 1 to base prio
     local degraded_prio=$(expr $DEGRADED_PRIO + 1)
+    local max_tries=10
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
@@ -159,8 +160,15 @@ function TEST_backfill_priority() {
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${chk_osd1_1}) dump_reservations || return 1
 
     # 3. Item is in progress, adjust priority with no higher priority waiting
-    while(ceph pg force-backfill $PG3 2>&1 | grep -q "doesn't require backfilling")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-backfill $PG3 2>&1 | grep -q "doesn't require backfilling"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-backfill"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     flush_pg_stats || return 1
@@ -202,8 +210,15 @@ function TEST_backfill_priority() {
     fi
 
     # 1. Item is queued, re-queue with new priority
-    while(ceph pg force-backfill $PG2 2>&1 | grep -q "doesn't require backfilling")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-backfill $PG2 2>&1 | grep -q "doesn't require backfilling"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-backfill"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     sleep 2

diff --git a/qa/standalone/osd/osd-recovery-prio.sh b/qa/standalone/osd/osd-recovery-prio.sh
index 25ecb2651248e44b5ee20be00bf4b75974baa36f..9b017800f674e55881f67391e6cca79fb49af8df 100755 (executable)
--- a/qa/standalone/osd/osd-recovery-prio.sh
+++ b/qa/standalone/osd/osd-recovery-prio.sh
@@ -44,6 +44,7 @@ function TEST_recovery_priority() {
     local dir=$1
     local pools=10
     local OSDS=5
+    local max_tries=10
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
@@ -154,8 +155,15 @@ function TEST_recovery_priority() {
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${chk_osd1_1}) dump_reservations || return 1
 
     # 3. Item is in progress, adjust priority with no higher priority waiting
-    while(ceph pg force-recovery $PG3 2>&1 | grep -q "doesn't require recovery")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-recovery $PG3 2>&1 | grep -q "doesn't require recovery"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-recovery"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     flush_pg_stats || return 1
@@ -197,8 +205,15 @@ function TEST_recovery_priority() {
     fi
 
     # 1. Item is queued, re-queue with new priority
-    while(ceph pg force-recovery $PG2 2>&1 | grep -q "doesn't require recovery")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-recovery $PG2 2>&1 | grep -q "doesn't require recovery"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-recovery"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     sleep 2