teardown $dir || return 1
}
+########################################################################
+##
+# Create a temporary file with random data and return its name. The file
+# should be removed by the caller.
+#
+# Example:
+# testdata=$(file_with_random_data 1024)
+#
+# @param the size of the file created in bytes. As in most use cases
+# the size is meaningless, a default (512) is used if not specified
+# @return the name of the file created
+#
+function file_with_random_data() {
+ local size_bytes=${1:-512}
+ local file=$(mktemp)
+ dd if=/dev/urandom of=$file bs=$size_bytes count=1
+ printf '%s' "$file"
+}
+
+function test_file_with_random_data() {
+ local dir=$1
+
+ setup $dir || return 1
+ local file=$(file_with_random_data 4000)
+ test -f $file || return 1
+ test $(stat -c %s $file) = 4000 || return 1
+ rm $file
+ teardown $dir || return 1
+}
+
+
#######################################################################
##
function TEST_divergent() {
local dir=$1
- # something that is always there
- local dummyfile='/etc/fstab'
- local dummyfile2='/etc/resolv.conf'
-
+ local dummyfile=$(file_with_random_data)
local num_osds=3
local osds="$(seq 0 $(expr $num_osds - 1))"
run_mon $dir a || return 1
# write a bunch of objects
for i in $(seq 1 $testobjects)
do
- rados -p $poolname put existing_$i $dummyfile
+ rados -p $poolname put existing_$i $dummyfile || return 1
done
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
echo "writing non-divergent object $objname"
ceph pg dump pgs
- rados -p $poolname put $objname $dummyfile2
+ # a second object (using a different size, for good measure)
+ dd if=/dev/urandom bs=1000 count=1 | rados -p "$poolname" put "$objname" - || return 1
# ensure no recovery of up osds first
echo 'delay recovery'
fi
echo "success"
+ rm -f $dummyfile
delete_pool $poolname
kill_daemons $dir || return 1
}
function TEST_divergent_ec() {
local dir=$1
- # something that is always there
- local dummyfile='/etc/fstab'
- local dummyfile2='/etc/resolv.conf'
+ local dummyfile=$(file_with_random_data)
+ # a second object, different in size and contents
+ local dummyfile2=$(file_with_random_data 1000)
local num_osds=3
local osds="$(seq 0 $(expr $num_osds - 1))"
# write a bunch of objects
for i in $(seq 1 $testobjects)
do
- rados -p $poolname put existing_$i $dummyfile
+ rados -p $poolname put existing_$i $dummyfile || return 1
done
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
echo "writing non-divergent object $objname"
ceph pg dump pgs
- rados -p $poolname put $objname $dummyfile2
+ rados -p $poolname put $objname $dummyfile2 || return 1
+ rm -f $dummyfile2
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
echo 'wait for peering'
ceph pg dump pgs
rados -p $poolname put foo $dummyfile
+ rm -f $dummyfile
echo "killing divergent $divergent"
ceph pg dump pgs
function TEST_divergent_2() {
local dir=$1
- # something that is always there
- local dummyfile='/etc/fstab'
- local dummyfile2='/etc/resolv.conf'
+ local dummyfile=$(file_with_random_data)
+ # a second object, different in size and contents
+ local dummyfile2=$(file_with_random_data 1000)
local num_osds=3
local osds="$(seq 0 $(expr $num_osds - 1))"
# write a bunch of objects
for i in $(seq 1 $testobjects)
do
- rados -p $poolname put existing_$i $dummyfile
+ rados -p $poolname put existing_$i $dummyfile || return 1
done
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
echo "writing non-divergent object $objname"
ceph pg dump pgs
- rados -p $poolname put $objname $dummyfile2
+ rados -p $poolname put $objname $dummyfile2 || return 1
+ rm -f $dummyfile2
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
ceph pg dump pgs
echo 'wait for peering'
ceph pg dump pgs
- rados -p $poolname put foo $dummyfile
+ rados -p $poolname put foo $dummyfile || return 1
+ rm -f $dummyfile
# At this point the divergent_priors should have been detected
echo "success"
rm $dir/$expfile
-
delete_pool $poolname
kill_daemons $dir || return 1
}
function TEST_divergent_3() {
local dir=$1
- # something that is always there
- local dummyfile='/etc/fstab'
- local dummyfile2='/etc/resolv.conf'
+ local dummyfile=$(file_with_random_data)
+ # a second file (using a different size, for good measure)
+ local dummyfile2=$(file_with_random_data 1000)
local num_osds=3
local osds="$(seq 0 $(expr $num_osds - 1))"
# write a bunch of objects
for i in $(seq 1 $testobjects)
do
- rados -p $poolname put existing_$i $dummyfile
+ rados -p $poolname put existing_$i $dummyfile || return 1
done
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
echo "writing non-divergent object $objname"
ceph pg dump pgs
- rados -p $poolname put $objname $dummyfile2
+ rados -p $poolname put $objname $dummyfile2 || return 1
WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
echo 'wait for peering'
ceph pg dump pgs
rados -p $poolname put foo $dummyfile
+ rm -f $dummyfile
+ rm -f $dummyfile2
# At this point the divergent_priors should have been detected
echo "success"
rm $dir/$expfile
-
delete_pool $poolname
kill_daemons $dir || return 1
}
local OSDS=3
local objects=15
- TESTDATA="testdata.$$"
-
run_mon $dir a --osd_pool_default_size=3 || return 1
run_mgr $dir x --mgr_stats_period=1 || return 1
local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
wait_for_clean || return 1
poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
- dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
+ local testdata_file=$(file_with_random_data 1032)
for i in `seq 1 $objects`
do
- rados -p $poolname put obj${i} $TESTDATA
+ rados -p $poolname put obj${i} $testdata_file || return 1
done
- rm -f $TESTDATA
+ rm -f $testdata_file
local primary=$(get_primary $poolname obj1)
local otherosd=$(get_not_primary $poolname obj1)
local anotherosd="2"
fi
- CORRUPT_DATA="corrupt-data.$$"
- dd if=/dev/urandom of=$CORRUPT_DATA bs=512 count=1
- objectstore_tool $dir $anotherosd obj1 set-bytes $CORRUPT_DATA
- rm -f $CORRUPT_DATA
+ local corrupt_data_file=$(file_with_random_data 512)
+ objectstore_tool $dir $anotherosd obj1 set-bytes $corrupt_data_file || return 1
+ rm -f $corrupt_data_file
local pgid="${poolid}.0"
pg_deep_scrub "$pgid" || return 1
ceph tell osd.* config set osd_scrub_min_interval 7200
ceph tell osd.* config set osd_deep_scrub_interval 14400
ceph tell osd.* config set osd_max_scrubs 32
- ceph tell osd.* config set osd_scrub_sleep 0
+ ceph tell osd.* config set osd_scrub_sleep 1
ceph tell osd.* config set osd_shallow_scrub_chunk_max 10
ceph tell osd.* config set osd_scrub_chunk_max 10
tout=20
while [ $tout -gt 0 ] ; do
- sleep 0.5
- (( extr_dbg >= 2 )) && ceph pg dump pgs --format=json-pretty | \
+ sleep 1
+ (( extr_dbg >= 1 )) && ceph pg dump pgs --format=json-pretty | \
jq '.pg_stats | map(select(.last_scrub_duration == 0)) | map({pgid: .pgid, last_scrub_duration: .last_scrub_duration})'
not_done=$(ceph pg dump pgs --format=json-pretty | \
jq '.pg_stats | map(select(.last_scrub_duration == 0)) | map({pgid: .pgid, last_scrub_duration: .last_scrub_duration})' | wc -l )
echo "Still waiting for $not_done PGs to finish initial scrubs (timeout $tout)"
tout=$((tout - 1))
done
+ ceph pg dump pgs --format=json-pretty | \
+ jq '.pg_stats | map(select(.last_scrub_duration == 0)) | map({pgid: .pgid, last_scrub_duration: .last_scrub_duration})'
(( tout == 0 )) && return 1
return 0
}
)
local extr_dbg=1 # note: 3 and above leave some temp files around
- standard_scrub_wpq_cluster "$dir" cluster_conf 3 || return 1
+ standard_scrub_wpq_cluster "$dir" cluster_conf 2 || return 1
local poolid=${cluster_conf['pool_id']}
local poolname=${cluster_conf['pool_name']}
echo "Pool: $poolname : $poolid"
projects / ceph.git / commitdiff