return $return_code
}
-# osd_scrub_during_recovery=true make sure scrub happens
-function TEST_recovery_scrub_2() {
- local dir=$1
- local poolname=test
-
- TESTDATA="testdata.$$"
- OSDS=8
- PGS=32
- OBJECTS=40
-
- setup $dir || return 1
- run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1
- run_mgr $dir x || return 1
- local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 "
- ceph_osd_args+="--osd_scrub_backoff_ratio=0 "
- ceph_osd_args+="--osd_stats_update_period_not_scrubbing=3 "
- ceph_osd_args+="--osd_stats_update_period_scrubbing=2"
- for osd in $(seq 0 $(expr $OSDS - 1))
- do
- run_osd $dir $osd --osd_scrub_during_recovery=true --osd_recovery_sleep=10 \
- $ceph_osd_args || return 1
- done
-
- # Create a pool with $PGS pgs
- create_pool $poolname $PGS $PGS
- wait_for_clean || return 1
- poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }')
-
- dd if=/dev/urandom of=$TESTDATA bs=1M count=50
- for i in $(seq 1 $OBJECTS)
- do
- rados -p $poolname put obj${i} $TESTDATA
- done
- rm -f $TESTDATA
-
- ceph osd pool set $poolname size 3
-
- ceph pg dump pgs
-
- # note that the following will be needed if the mclock scheduler is specified
- #ceph tell osd.* config get osd_mclock_override_recovery_settings
-
- # the '_max_active' is expected to be 0
- ceph tell osd.1 config get osd_recovery_max_active
- # both next parameters are expected to be >=3
- ceph tell osd.1 config get osd_recovery_max_active_hdd
- ceph tell osd.1 config get osd_recovery_max_active_ssd
-
- # Wait for recovery to start
- count=0
- while(true)
- do
- #ceph --format json pg dump pgs | jq '.pg_stats | [.[].state]'
- if test $(ceph --format json pg dump pgs |
- jq '.pg_stats | [.[].state]'| grep recovering | wc -l) -ge 2
- then
- break
- fi
- sleep 2
- if test "$count" -eq "10"
- then
- echo "Not enough recovery started simultaneously"
- return 1
- fi
- count=$(expr $count + 1)
- done
- ceph pg dump pgs
-
- pids=""
- recov_scrub_count=0
- for pg in $(seq 0 $(expr $PGS - 1))
- do
- run_in_background pids pg_scrub_mod $poolid.$(printf "%x" $pg)
- done
- wait_background_check pids
- return_code=$?
- if [ $return_code -ne 0 ]; then return $return_code; fi
-
- ERRORS=0
- if test $recov_scrub_count -eq 0
- then
- echo "No scrubs occurred while PG recovering"
- ERRORS=$(expr $ERRORS + 1)
- fi
-
- pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid')
- pid=$(cat $pidfile)
- if ! kill -0 $pid
- then
- echo "OSD crash occurred"
- #tail -100 $dir/osd.0.log
- ERRORS=$(expr $ERRORS + 1)
- fi
-
- # Work around for http://tracker.ceph.com/issues/38195
- kill_daemons $dir #|| return 1
-
- declare -a err_strings
- err_strings[0]="not scheduling scrubs due to active recovery"
-
- for osd in $(seq 0 $(expr $OSDS - 1))
- do
- grep "not scheduling scrubs" $dir/osd.${osd}.log
- done
- for err_string in "${err_strings[@]}"
- do
- found=false
- for osd in $(seq 0 $(expr $OSDS - 1))
- do
- if grep "$err_string" $dir/osd.${osd}.log > /dev/null;
- then
- found=true
- fi
- done
- if [ "$found" = "true" ]; then
- echo "Found log message not expected '$err_string'"
- ERRORS=$(expr $ERRORS + 1)
- fi
- done
-
- teardown $dir || return 1
-
- if [ $ERRORS != "0" ];
- then
- echo "TEST FAILED WITH $ERRORS ERRORS"
- return 1
- fi
-
- echo "TEST PASSED"
- return 0
-}
-
main osd-recovery-scrub "$@"
# Local Variables: