From: Jason Dillaman Date: Wed, 15 Jun 2016 21:49:54 +0000 (-0400) Subject: qa/workunits/rbd: increase writes in large image count test X-Git-Tag: v10.2.3~53^2~3 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=c2e6d08469ea733dac984770c30d8ab6356f3eca;p=ceph.git qa/workunits/rbd: increase writes in large image count test This will help to test edge cases where the remote image does or does not own the exclusive lock when the sync starts. Signed-off-by: Jason Dillaman (cherry picked from commit d16698f96c65b9627cc1f64ef80b7f1e39d69d45) --- diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 6a24d0ec5ab28..9478cc8f9d1c2 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -408,14 +408,28 @@ wait_for_replay_complete() local cluster=$2 local pool=$3 local image=$4 - local s master_pos mirror_pos - - for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do - sleep ${s} - flush "${local_cluster}" "${pool}" "${image}" - master_pos=$(get_master_position "${cluster}" "${pool}" "${image}") - mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}") - test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + local s master_pos mirror_pos last_mirror_pos + local master_tag master_entry mirror_tag mirror_entry + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + flush "${local_cluster}" "${pool}" "${image}" + master_pos=$(get_master_position "${cluster}" "${pool}" "${image}") + mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}") + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + test "${mirror_pos}" != "${last_mirror_pos}" && break + done + + test "${mirror_pos}" = "${last_mirror_pos}" && return 1 + last_mirror_pos="${mirror_pos}" + + # handle the case where the mirror is ahead of the master + master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0 done return 1 } diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index a0e34913066b8..1caea23326074 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -2,6 +2,11 @@ # # rbd_mirror_stress.sh - stress test rbd-mirror daemon # +# The following additional environment variables affect the test: +# +# RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write +# tool during the many image test +# IMAGE_COUNT=50 export LOCKDEP=0 @@ -40,23 +45,43 @@ compare_image_snaps() rm -f ${rmt_export} ${loc_export} } -wait_for_pool_healthy() +wait_for_pool_images() { local cluster=$1 local pool=$2 local image_count=$3 local s local count + local last_count=0 + + while true; do + for s in `seq 1 40`; do + count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2) + test "${count}" = "${image_count}" && return 0 + + # reset timeout if making forward progress + test $count -gt $last_count && break + sleep 30 + done + + test $count -eq $last_count && return 1 + $last_count=$count + done + return 1 +} + +wait_for_pool_healthy() +{ + local cluster=$1 + local pool=$2 + local s local state for s in `seq 1 40`; do + state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2) + test "${state}" = "ERROR" && return 1 + test "${state}" = "OK" && return 0 sleep 30 - count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ') - test "${count}" = "images: ${image_count} total" || continue - - state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:') - test "${state}" = "health: ERROR" && return 1 - test "${state}" = "health: OK" && return 0 done return 1 } @@ -91,29 +116,40 @@ remove_image ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' testlog "TEST: create many images" +snap_name="snap" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" create_image ${CLUSTER2} ${POOL} ${image} '128M' - write_image ${CLUSTER2} ${POOL} ${image} 100 + if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then + write_image ${CLUSTER2} ${POOL} ${image} 100 + else + stress_write_image ${CLUSTER2} ${POOL} ${image} + fi done -wait_for_pool_healthy ${CLUSTER2} ${POOL} ${IMAGE_COUNT} -wait_for_pool_healthy ${CLUSTER1} ${POOL} ${IMAGE_COUNT} +wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER2} ${POOL} + +wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER1} ${POOL} testlog "TEST: compare many images" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" + create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} - compare_images ${POOL} ${image} + wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} + compare_image_snaps ${POOL} ${image} ${snap_name} done testlog "TEST: delete many images" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" + remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} remove_image ${CLUSTER2} ${POOL} ${image} done