From d16698f96c65b9627cc1f64ef80b7f1e39d69d45 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 15 Jun 2016 17:49:54 -0400 Subject: [PATCH] qa/workunits/rbd: increase writes in large image count test This will help to test edge cases where the remote image does or does not own the exclusive lock when the sync starts. Signed-off-by: Jason Dillaman --- qa/workunits/rbd/rbd_mirror_helpers.sh | 30 +++++++++---- qa/workunits/rbd/rbd_mirror_stress.sh | 58 +++++++++++++++++++++----- 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 8094bd8de427e..3e425fac1237d 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -408,14 +408,28 @@ wait_for_replay_complete() local cluster=$2 local pool=$3 local image=$4 - local s master_pos mirror_pos - - for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do - sleep ${s} - flush "${local_cluster}" "${pool}" "${image}" - master_pos=$(get_master_position "${cluster}" "${pool}" "${image}") - mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}") - test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + local s master_pos mirror_pos last_mirror_pos + local master_tag master_entry mirror_tag mirror_entry + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + flush "${local_cluster}" "${pool}" "${image}" + master_pos=$(get_master_position "${cluster}" "${pool}" "${image}") + mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}") + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + test "${mirror_pos}" != "${last_mirror_pos}" && break + done + + test "${mirror_pos}" = "${last_mirror_pos}" && return 1 + last_mirror_pos="${mirror_pos}" + + # handle the case where the mirror is ahead of the master + master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0 done return 1 } diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index a8561eed9d8d7..a2565d4886a7d 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -2,6 +2,11 @@ # # rbd_mirror_stress.sh - stress test rbd-mirror daemon # +# The following additional environment variables affect the test: +# +# RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write +# tool during the many image test +# IMAGE_COUNT=50 export LOCKDEP=0 @@ -39,23 +44,43 @@ compare_image_snaps() cmp ${rmt_export} ${loc_export} } -wait_for_pool_healthy() +wait_for_pool_images() { local cluster=$1 local pool=$2 local image_count=$3 local s local count + local last_count=0 + + while true; do + for s in `seq 1 40`; do + count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2) + test "${count}" = "${image_count}" && return 0 + + # reset timeout if making forward progress + test $count -gt $last_count && break + sleep 30 + done + + test $count -eq $last_count && return 1 + $last_count=$count + done + return 1 +} + +wait_for_pool_healthy() +{ + local cluster=$1 + local pool=$2 + local s local state for s in `seq 1 40`; do + state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2) + test "${state}" = "ERROR" && return 1 + test "${state}" = "OK" && return 0 sleep 30 - count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ') - test "${count}" = "images: ${image_count} total" || continue - - state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:') - test "${state}" = "health: ERROR" && return 1 - test "${state}" = "health: OK" && return 0 done return 1 } @@ -90,29 +115,40 @@ remove_image ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' testlog "TEST: create many images" +snap_name="snap" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" create_image ${CLUSTER2} ${POOL} ${image} '128M' - write_image ${CLUSTER2} ${POOL} ${image} 100 + if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then + write_image ${CLUSTER2} ${POOL} ${image} 100 + else + stress_write_image ${CLUSTER2} ${POOL} ${image} + fi done -wait_for_pool_healthy ${CLUSTER2} ${POOL} ${IMAGE_COUNT} -wait_for_pool_healthy ${CLUSTER1} ${POOL} ${IMAGE_COUNT} +wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER2} ${POOL} + +wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER1} ${POOL} testlog "TEST: compare many images" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" + create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} - compare_images ${POOL} ${image} + wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} + compare_image_snaps ${POOL} ${image} ${snap_name} done testlog "TEST: delete many images" for i in `seq 1 ${IMAGE_COUNT}` do image="image_${i}" + remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} remove_image ${CLUSTER2} ${POOL} ${image} done -- 2.39.5