From: David Zafman <dzafman@redhat.com>
Date: Wed, 13 Feb 2019 18:13:12 +0000 (-0800)
Subject: test: Limit loops waiting for force-backfill/force-recovery to happen
X-Git-Tag: v14.1.0~92^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=64beabc4c62017ce5df41852c62d143f67dff3ad;p=ceph.git

test: Limit loops waiting for force-backfill/force-recovery to happen

Fixes: http://tracker.ceph.com/issues/38309

Signed-off-by: David Zafman <dzafman@redhat.com>
---

diff --git a/qa/standalone/osd/osd-backfill-prio.sh b/qa/standalone/osd/osd-backfill-prio.sh
index 248ac6fb0c27..6491cd19efd6 100755
--- a/qa/standalone/osd/osd-backfill-prio.sh
+++ b/qa/standalone/osd/osd-backfill-prio.sh
@@ -48,6 +48,7 @@ function TEST_backfill_priority() {
     local OSDS=5
     # size 2 -> 1 means degraded by 1, so add 1 to base prio
     local degraded_prio=$(expr $DEGRADED_PRIO + 1)
+    local max_tries=10
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
@@ -159,8 +160,15 @@ function TEST_backfill_priority() {
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${chk_osd1_1}) dump_reservations || return 1
 
     # 3. Item is in progress, adjust priority with no higher priority waiting
-    while(ceph pg force-backfill $PG3 2>&1 | grep -q "doesn't require backfilling")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-backfill $PG3 2>&1 | grep -q "doesn't require backfilling"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-backfill"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     flush_pg_stats || return 1
@@ -202,8 +210,15 @@ function TEST_backfill_priority() {
     fi
 
     # 1. Item is queued, re-queue with new priority
-    while(ceph pg force-backfill $PG2 2>&1 | grep -q "doesn't require backfilling")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-backfill $PG2 2>&1 | grep -q "doesn't require backfilling"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-backfill"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     sleep 2
diff --git a/qa/standalone/osd/osd-recovery-prio.sh b/qa/standalone/osd/osd-recovery-prio.sh
index 25ecb2651248..9b017800f674 100755
--- a/qa/standalone/osd/osd-recovery-prio.sh
+++ b/qa/standalone/osd/osd-recovery-prio.sh
@@ -44,6 +44,7 @@ function TEST_recovery_priority() {
     local dir=$1
     local pools=10
     local OSDS=5
+    local max_tries=10
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
@@ -154,8 +155,15 @@ function TEST_recovery_priority() {
     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${chk_osd1_1}) dump_reservations || return 1
 
     # 3. Item is in progress, adjust priority with no higher priority waiting
-    while(ceph pg force-recovery $PG3 2>&1 | grep -q "doesn't require recovery")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-recovery $PG3 2>&1 | grep -q "doesn't require recovery"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-recovery"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     flush_pg_stats || return 1
@@ -197,8 +205,15 @@ function TEST_recovery_priority() {
     fi
 
     # 1. Item is queued, re-queue with new priority
-    while(ceph pg force-recovery $PG2 2>&1 | grep -q "doesn't require recovery")
+    for i in $(seq 1 $max_tries)
     do
+      if ! ceph pg force-recovery $PG2 2>&1 | grep -q "doesn't require recovery"; then
+        break
+      fi
+      if [ "$i" = "$max_tries" ]; then
+        echo "ERROR: Didn't appear to be able to force-recovery"
+        ERRORS=$(expr $ERRORS + 1)
+      fi
       sleep 2
     done
     sleep 2