From f1becf9ad7237f36cf65e2b8dc95ee43946fe1fd Mon Sep 17 00:00:00 2001
From: Loic Dachary <loic-201408@dachary.org>
Date: Sat, 4 Oct 2014 11:34:27 +0200
Subject: [PATCH] qa: ceph tell must retry on ENXIO

It is expected for ceph tell to fail with ENXIO if the daemon it is
trying to join is not ready for some reason. This should be handled as a
transient error instead of a fatal error.

Add two shell functions to help with retry. They may prove useful if
other cases requiring a few retries show up.

http://tracker.ceph.com/issues/9655 Fixes: #9655

Signed-off-by: Loic Dachary <loic-201408@dachary.org>
---
 qa/workunits/cephtool/test.sh | 53 ++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index c9c68f6f57e3..1f01f5d36236 100755
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -52,6 +52,57 @@ trap "rm -fr $TMPDIR" 0
 
 TMPFILE=$TMPDIR/test_invalid.$$
 
+#
+# retry_eagain max cmd args ...
+#
+# retry cmd args ... if it exits on error and its output contains the
+# string EAGAIN, at most $max times
+#
+function retry_eagain()
+{
+    local max=$1
+    shift
+    local status
+    local tmpfile=$TMPDIR/retry_eagain.$$
+    local count
+    for count in $(seq 1 $max) ; do
+        status=0
+        "$@" > $tmpfile 2>&1 || status=$?
+        if test $status = 0 || 
+            ! grep --quiet EAGAIN $tmpfile ; then
+            break
+        fi
+        sleep 1
+    done
+    if test $count = $max ; then
+        echo retried with non zero exit status, $max times: "$@" >&2
+    fi
+    cat $tmpfile
+    rm $tmpfile
+    return $status
+}
+
+#
+# map_enxio_to_eagain cmd arg ...
+#
+# add EAGAIN to the output of cmd arg ... if the output contains
+# ENXIO.
+#
+function map_enxio_to_eagain()
+{
+    local status=0
+    local tmpfile=$TMPDIR/map_enxio_to_eagain.$$
+
+    "$@" > $tmpfile 2>&1 || status=$?
+    if test $status != 0 &&
+        grep --quiet ENXIO $tmpfile ; then
+        echo "EAGAIN added by $0::map_enxio_to_eagain" >> $tmpfile
+    fi
+    cat $tmpfile
+    rm $tmpfile
+    return $status
+}
+
 function check_response()
 {
 	expected_stderr_string=$1
@@ -633,7 +684,7 @@ function test_mon_osd()
   ceph osd getmaxosd | grep "max_osd = $save"
 
   for id in `ceph osd ls` ; do
-    ceph tell osd.$id version
+    retry_eagain 5 map_enxio_to_eagain ceph tell osd.$id version
   done
 
   ceph osd rm 0 2>&1 | grep 'EBUSY'
-- 
2.47.3