From 0186dbc8265e353f20b0943b205e827ffda13d3b Mon Sep 17 00:00:00 2001 From: David Zafman Date: Tue, 1 Aug 2017 21:35:34 -0700 Subject: [PATCH] qa: Add common generalized inject_eio() to ceph-helpers.sh Retry for a while to allow pool to appear Signed-off-by: David Zafman (cherry picked from commit 574b3cd3d416a3ba3611775f871646ab137266f5) --- qa/standalone/ceph-helpers.sh | 31 +++++++++++++++++++ .../erasure-code/test-erasure-eio.sh | 20 ++---------- qa/standalone/scrub/osd-scrub-repair.sh | 28 +++++------------ 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh index 669b9fa16e4f5..42d1998898d94 100755 --- a/qa/standalone/ceph-helpers.sh +++ b/qa/standalone/ceph-helpers.sh @@ -1925,6 +1925,37 @@ function jq_success() { return 1 } +function inject_eio() { + local pooltype=$1 + shift + local which=$1 + shift + local poolname=$1 + shift + local objname=$1 + shift + local dir=$1 + shift + local shard_id=$1 + shift + + local -a initial_osds=($(get_osds $poolname $objname)) + local osd_id=${initial_osds[$shard_id]} + if [ "$pooltype" != "ec" ]; then + shard_id="" + fi + set_config osd $osd_id filestore_debug_inject_read_err true || return 1 + local loop=0 + while ( CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.$osd_id) \ + inject${which}err $poolname $objname $shard_id | grep -q Invalid ); do + loop=$(expr $loop + 1) + if [ $loop = "10" ]; then + return 1 + fi + sleep 1 + done +} + # Local Variables: # compile-command: "cd ../../src ; make -j4 && ../qa/standalone/ceph-helpers.sh TESTS # test_get_config" # End: diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh index b26702d542305..b7880169ede07 100755 --- a/qa/standalone/erasure-code/test-erasure-eio.sh +++ b/qa/standalone/erasure-code/test-erasure-eio.sh @@ -142,22 +142,6 @@ function rados_put_get() { rm $dir/ORIGINAL } -function inject_eio() { - local objname=$1 - shift - local dir=$1 - shift - local shard_id=$1 - shift - - local poolname=pool-jerasure - local -a initial_osds=($(get_osds $poolname $objname)) - local osd_id=${initial_osds[$shard_id]} - set_config osd $osd_id filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.$osd_id) \ - injectdataerr $poolname $objname $shard_id || return 1 -} - function rados_get_data_eio() { local dir=$1 shift @@ -170,11 +154,11 @@ function rados_get_data_eio() { # local poolname=pool-jerasure local objname=obj-eio-$$-$shard_id - inject_eio $objname $dir $shard_id || return 1 + inject_eio ec data $poolname $objname $dir $shard_id || return 1 rados_put_get $dir $poolname $objname $recovery || return 1 shard_id=$(expr $shard_id + 1) - inject_eio $objname $dir $shard_id || return 1 + inject_eio ec data $poolname $objname $dir $shard_id || return 1 # Now 2 out of 3 shards get EIO, so should fail rados_get $dir $poolname $objname fail || return 1 } diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh index 64f50b3da0274..2af4ea7996348 100755 --- a/qa/standalone/scrub/osd-scrub-repair.sh +++ b/qa/standalone/scrub/osd-scrub-repair.sh @@ -603,16 +603,10 @@ function TEST_corrupt_scrub_replicated() { local pg=$(get_pg $poolname ROBJ0) - set_config osd 0 filestore_debug_inject_read_err true || return 1 - set_config osd 1 filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ11 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ12 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ13 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ13 || return 1 + inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 + inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 + inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 + inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 pg_scrub $pg @@ -975,16 +969,10 @@ EOF objectstore_tool $dir 1 $objname set-attr _ $dir/oi rm $dir/oi - set_config osd 0 filestore_debug_inject_read_err true || return 1 - set_config osd 1 filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ11 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ12 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ13 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ13 || return 1 + inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 + inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 + inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 + inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 pg_deep_scrub $pg rados list-inconsistent-pg $poolname > $dir/json || return 1 -- 2.39.5