qa: Add common generalized inject_eio() to ceph-helpers.sh

Retry for a while to allow pool to appear

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 574b3cd3d416a3ba3611775f871646ab137266f5)

Conflicts:
	qa/workunits/ceph-helpers.sh (trivial)
	src/test/erasure-code/test-erasure-eio.sh (trivial)
	src/test/osd/osd-scrub-repair.sh (asok sockets not in /tmp)

diff --git a/qa/workunits/ceph-helpers.sh b/qa/workunits/ceph-helpers.sh
index 87f8f6ef139546e01474972d35e7e4582b0f9eb9..9437072d30c865ba7f04736a9d060b61e27a82f1 100755 (executable)
--- a/qa/workunits/ceph-helpers.sh
+++ b/qa/workunits/ceph-helpers.sh
@@ -1504,6 +1504,37 @@ if test "$1" = TESTS ; then
     run_tests "$@"
 fi
 
+function inject_eio() {
+    local pooltype=$1
+    shift
+    local which=$1
+    shift
+    local poolname=$1
+    shift
+    local objname=$1
+    shift
+    local dir=$1
+    shift
+    local shard_id=$1
+    shift
+
+    local -a initial_osds=($(get_osds $poolname $objname))
+    local osd_id=${initial_osds[$shard_id]}
+    if [ "$pooltype" != "ec" ]; then
+        shard_id=""
+    fi
+    set_config osd $osd_id filestore_debug_inject_read_err true || return 1
+    local loop=0
+    while ( CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.$osd_id.asok \
+             inject${which}err $poolname $objname $shard_id | grep -q Invalid ); do
+        loop=$(expr $loop + 1)
+        if [ $loop = "10" ]; then
+            return 1
+        fi
+        sleep 1
+    done
+}
+
 # Local Variables:
 # compile-command: "cd ../../src ; make -j4 && ../qa/workunits/ceph-helpers.sh TESTS # test_get_config"
 # End:

diff --git a/src/test/erasure-code/test-erasure-eio.sh b/src/test/erasure-code/test-erasure-eio.sh
index 46a7bfcfd0ba716f2fb3493360aa689c18fe1042..2571c81548641f123d1b453f0dcd449f81b9a891 100755 (executable)
--- a/src/test/erasure-code/test-erasure-eio.sh
+++ b/src/test/erasure-code/test-erasure-eio.sh
@@ -139,22 +139,6 @@ function rados_put_get() {
     rm $dir/ORIGINAL
 }
 
-function inject_eio() {
-    local objname=$1
-    shift
-    local dir=$1
-    shift
-    local shard_id=$1
-    shift
-
-    local poolname=pool-jerasure
-    local -a initial_osds=($(get_osds $poolname $objname))
-    local osd_id=${initial_osds[$shard_id]}
-    set_config osd $osd_id filestore_debug_inject_read_err true || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.$osd_id.asok \
-             injectdataerr $poolname $objname $shard_id || return 1
-}
-
 function rados_get_data_eio() {
     local dir=$1
     shift
@@ -167,11 +151,11 @@ function rados_get_data_eio() {
     #
     local poolname=pool-jerasure
     local objname=obj-eio-$$-$shard_id
-    inject_eio $objname $dir $shard_id || return 1
+    inject_eio ec data $poolname $objname $dir $shard_id || return 1
     rados_put_get $dir $poolname $objname $recovery || return 1
 
     shard_id=$(expr $shard_id + 1)
-    inject_eio $objname $dir $shard_id || return 1
+    inject_eio ec data $poolname $objname $dir $shard_id || return 1
     # Now 2 out of 3 shards get EIO, so should fail
     rados_get $dir $poolname $objname fail || return 1
 }

diff --git a/src/test/osd/osd-scrub-repair.sh b/src/test/osd/osd-scrub-repair.sh
index 517e795e7fc8bd42706e8d23cbee09d5c9d4d6b2..ba0655659619c76319aeb8df55abba9b0afd2b88 100755 (executable)
--- a/src/test/osd/osd-scrub-repair.sh
+++ b/src/test/osd/osd-scrub-repair.sh
@@ -505,16 +505,10 @@ function TEST_corrupt_scrub_replicated() {
 
     local pg=$(get_pg $poolname ROBJ0)
 
-    set_config osd 0 filestore_debug_inject_read_err true || return 1
-    set_config osd 1 filestore_debug_inject_read_err true || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
-             injectdataerr $poolname ROBJ11 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
-             injectmdataerr $poolname ROBJ12 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
-             injectmdataerr $poolname ROBJ13 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
-             injectdataerr $poolname ROBJ13 || return 1
+    inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
+    inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
 
     pg_scrub $pg
 
@@ -877,16 +871,10 @@ EOF
     objectstore_tool $dir 1 $objname set-attr _ $dir/oi
     rm $dir/oi
 
-    set_config osd 0 filestore_debug_inject_read_err true || return 1
-    set_config osd 1 filestore_debug_inject_read_err true || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
-             injectdataerr $poolname ROBJ11 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
-             injectmdataerr $poolname ROBJ12 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
-             injectmdataerr $poolname ROBJ13 || return 1
-    CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
-             injectdataerr $poolname ROBJ13 || return 1
+    inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
+    inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
+    inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
     pg_deep_scrub $pg
 
     rados list-inconsistent-pg $poolname > $dir/json || return 1