From d6f4ce2a386c9f21dbfbd91b8fd924bc48bfb9b4 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 26 Feb 2020 22:25:54 -0500 Subject: [PATCH] qa: test cases for snapshot-based mirroring Signed-off-by: Jason Dillaman --- .../rbd-mirror-snapshot-workunit.yaml | 12 + qa/workunits/rbd/rbd_mirror_helpers.sh | 397 +++++++++------ qa/workunits/rbd/rbd_mirror_journal.sh | 16 +- qa/workunits/rbd/rbd_mirror_snapshot.sh | 454 ++++++++++++++++++ qa/workunits/rbd/rbd_mirror_stress.sh | 2 +- 5 files changed, 725 insertions(+), 156 deletions(-) create mode 100644 qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit.yaml create mode 100755 qa/workunits/rbd/rbd_mirror_snapshot.sh diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit.yaml new file mode 100644 index 0000000000000..61b28ca53b86d --- /dev/null +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit.yaml @@ -0,0 +1,12 @@ +meta: +- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon +tasks: +- workunit: + clients: + cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh] + env: + # override workunit setting of CEPH_ARGS='--cluster' + CEPH_ARGS: '' + RBD_MIRROR_INSTANCES: '4' + RBD_MIRROR_USE_EXISTING_CLUSTER: '1' + RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index ab20e9f75bfa3..8eaa6e8e215d2 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -87,6 +87,9 @@ NS2=ns2 TEMPDIR= CEPH_ID=${CEPH_ID:-mirror} MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} +MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool} +MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal} + export CEPH_ARGS="--id ${CEPH_ID}" LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1)) @@ -128,8 +131,8 @@ set_cluster_instance() instance=${val##*:} if [ "${instance}" = "${val}" ]; then - # instance was not specified, use default - instance=0 + # instance was not specified, use default + instance=0 fi eval ${cluster_var_name}=${cluster} @@ -173,7 +176,7 @@ expect_failure() fi if [ -z "${expected}" ]; then - return 0 + return 0 fi if ! grep -q "${expected}" ${out} ; then @@ -249,13 +252,13 @@ setup_pools() fi CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable \ - --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} pool + --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} ${MIRROR_POOL_MODE} rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image rbd --cluster ${cluster} namespace create ${POOL}/${NS1} rbd --cluster ${cluster} namespace create ${POOL}/${NS2} - rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} pool + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then @@ -286,12 +289,12 @@ setup_pools() setup_tempdir() { if [ -n "${RBD_MIRROR_TEMDIR}" ]; then - test -d "${RBD_MIRROR_TEMDIR}" || - mkdir "${RBD_MIRROR_TEMDIR}" - TEMPDIR="${RBD_MIRROR_TEMDIR}" - cd ${TEMPDIR} + test -d "${RBD_MIRROR_TEMDIR}" || + mkdir "${RBD_MIRROR_TEMDIR}" + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} else - TEMPDIR=`mktemp -d` + TEMPDIR=`mktemp -d` fi } @@ -302,8 +305,8 @@ setup() setup_tempdir if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then - setup_cluster "${CLUSTER1}" - setup_cluster "${CLUSTER2}" + setup_cluster "${CLUSTER1}" + setup_cluster "${CLUSTER2}" fi setup_pools "${CLUSTER1}" "${CLUSTER2}" @@ -336,7 +339,7 @@ cleanup() CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it for cluster in "${CLUSTER1}" "${CLUSTER2}"; do - stop_mirrors "${cluster}" + stop_mirrors "${cluster}" done if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then @@ -366,16 +369,16 @@ start_mirror() test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return rbd-mirror \ - --cluster ${cluster} \ + --cluster ${cluster} \ --id ${MIRROR_USER_ID_PREFIX}${instance} \ - --rbd-mirror-delete-retry-interval=5 \ - --rbd-mirror-image-state-check-interval=5 \ - --rbd-mirror-journal-poll-age=1 \ - --rbd-mirror-pool-replayers-refresh-interval=5 \ - --debug-rbd=30 --debug-journaler=30 \ - --debug-rbd_mirror=30 \ - --daemonize=true \ - ${RBD_MIRROR_ARGS} + --rbd-mirror-delete-retry-interval=5 \ + --rbd-mirror-image-state-check-interval=5 \ + --rbd-mirror-journal-poll-age=1 \ + --rbd-mirror-pool-replayers-refresh-interval=5 \ + --debug-rbd=30 --debug-journaler=30 \ + --debug-rbd_mirror=30 \ + --daemonize=true \ + ${RBD_MIRROR_ARGS} } start_mirrors() @@ -398,12 +401,12 @@ stop_mirror() pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || : if [ -n "${pid}" ] then - kill ${sig} ${pid} - for s in 1 2 4 8 16 32; do - sleep $s - ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break - done - ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' + kill ${sig} ${pid} + for s in 1 2 4 8 16 32; do + sleep $s + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break + done + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' fi rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}") rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}") @@ -441,14 +444,14 @@ admin_daemons() local loop_instance for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do - sleep ${s} - if [ "${instance}" != "${cluster_instance}" ]; then - admin_daemon "${cluster}:${instance}" $@ && return 0 - else - for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do - admin_daemon "${cluster}:${loop_instance}" $@ && return 0 - done - fi + sleep ${s} + if [ "${instance}" != "${cluster_instance}" ]; then + admin_daemon "${cluster}:${instance}" $@ && return 0 + else + for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${loop_instance}" $@ && return 0 + done + fi done return 1 } @@ -458,7 +461,7 @@ all_admin_daemons() local cluster=$1 ; shift for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do - admin_daemon "${cluster}:${instance}" $@ + admin_daemon "${cluster}:${instance}" $@ done } @@ -468,92 +471,92 @@ status() for cluster in ${CLUSTER1} ${CLUSTER2} do - echo "${cluster} status" - CEPH_ARGS='' ceph --cluster ${cluster} -s - CEPH_ARGS='' ceph --cluster ${cluster} service dump - CEPH_ARGS='' ceph --cluster ${cluster} service status - echo - - for image_pool in ${POOL} ${PARENT_POOL} - do - for image_ns in "" "/${NS1}" "/${NS2}" + echo "${cluster} status" + CEPH_ARGS='' ceph --cluster ${cluster} -s + CEPH_ARGS='' ceph --cluster ${cluster} service dump + CEPH_ARGS='' ceph --cluster ${cluster} service status + echo + + for image_pool in ${POOL} ${PARENT_POOL} + do + for image_ns in "" "${NS1}" "${NS2}" do - echo "${cluster} ${image_pool}${image_ns} images" - rbd --cluster ${cluster} -p ${image_pool}{$image_ns} ls -l - echo + echo "${cluster} ${image_pool} ${image_ns} images" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls -l + echo echo "${cluster} ${image_pool}${image_ns} mirror pool info" - rbd --cluster ${cluster} -p ${image_pool}${image_ns} mirror pool info - echo - - echo "${cluster} ${image_pool}${image_ns} mirror pool status" - CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool}${image_ns} mirror pool status --verbose - echo - - for image in `rbd --cluster ${cluster} -p ${image_pool}${image_ns} ls 2>/dev/null` - do - echo "image ${image} info" - rbd --cluster ${cluster} -p ${image_pool}${image_ns} info ${image} - echo - echo "image ${image} journal status" - rbd --cluster ${cluster} -p ${image_pool}${image_ns} journal status --image ${image} - echo - done - - echo "${cluster} ${image_pool}${image_ns} rbd_mirroring omap vals" - rados --cluster ${cluster} -p ${image_pool}${image_ns} listomapvals rbd_mirroring + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool info + echo + + echo "${cluster} ${image_pool}${image_ns} mirror pool status" + CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool status --verbose + echo + + for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null` + do + echo "image ${image} info" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" info ${image} + echo + echo "image ${image} journal status" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" journal status --image ${image} + echo + done + + echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring echo done - done + done done local ret for cluster in "${CLUSTER1}" "${CLUSTER2}" do - for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do - local pid_file=$(daemon_pid_file ${cluster}:${instance}) - if [ ! -e ${pid_file} ] - then - echo "${cluster} rbd-mirror not running or unknown" \ - "(${pid_file} not exist)" - continue - fi - - local pid - pid=$(cat ${pid_file} 2>/dev/null) || : - if [ -z "${pid}" ] - then - echo "${cluster} rbd-mirror not running or unknown" \ - "(can't find pid using ${pid_file})" - ret=1 - continue - fi - - echo "${daemon} rbd-mirror process in ps output:" - if ps auxww | - awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' - then - echo - echo "${cluster} rbd-mirror not running" \ - "(can't find pid $pid in ps output)" - ret=1 - continue - fi - echo - - local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) - if [ ! -S "${asok_file}" ] - then - echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" - ret=1 - continue - fi - - echo "${cluster} rbd-mirror status" - ceph --admin-daemon ${asok_file} rbd mirror status - echo - done + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + local pid_file=$(daemon_pid_file ${cluster}:${instance}) + if [ ! -e ${pid_file} ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(${pid_file} not exist)" + continue + fi + + local pid + pid=$(cat ${pid_file} 2>/dev/null) || : + if [ -z "${pid}" ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(can't find pid using ${pid_file})" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror process in ps output:" + if ps auxww | + awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' + then + echo + echo "${cluster} rbd-mirror not running" \ + "(can't find pid $pid in ps output)" + ret=1 + continue + fi + echo + + local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) + if [ ! -S "${asok_file}" ] + then + echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" + ret=1 + continue + fi + + echo "${cluster} rbd-mirror status" + ceph --admin-daemon ${asok_file} rbd mirror status + echo + done done return ${ret} @@ -598,8 +601,8 @@ wait_for_image_replay_state() # TODO: add a way to force rbd-mirror to update replayers for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do - sleep ${s} - test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0 + sleep ${s} + test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0 done return 1 } @@ -622,7 +625,7 @@ wait_for_image_replay_stopped() wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped } -get_position() +get_journal_position() { local cluster=$1 local pool=$2 @@ -634,30 +637,30 @@ get_position() local status_log=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.status) rbd --cluster ${cluster} journal status --image ${pool}/${image} | - tee ${status_log} >&2 + tee ${status_log} >&2 sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \ - ${status_log} + ${status_log} } -get_master_position() +get_master_journal_position() { local cluster=$1 local pool=$2 local image=$3 - get_position "${cluster}" "${pool}" "${image}" '' + get_journal_position "${cluster}" "${pool}" "${image}" '' } -get_mirror_position() +get_mirror_journal_position() { local cluster=$1 local pool=$2 local image=$3 - get_position "${cluster}" "${pool}" "${image}" '..*' + get_journal_position "${cluster}" "${pool}" "${image}" '..*' } -wait_for_replay_complete() +wait_for_journal_replay_complete() { local local_cluster=$1 local cluster=$2 @@ -668,11 +671,11 @@ wait_for_replay_complete() while true; do for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do - sleep ${s} - flush "${local_cluster}" "${pool}" "${image}" - master_pos=$(get_master_position "${cluster}" "${pool}" "${image}") - mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}") - test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + sleep ${s} + flush "${local_cluster}" "${pool}" "${image}" + master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}") + mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}") + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 test "${mirror_pos}" != "${last_mirror_pos}" && break done @@ -689,6 +692,73 @@ wait_for_replay_complete() return 1 } +mirror_image_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster "${cluster}" mirror image snapshot "${pool}/${image}" +} + +get_newest_mirror_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local log=$4 + + rbd --cluster "${cluster}" snap list --all "${pool}/${image}" --format xml | \ + xmlstarlet sel -t -c "//snapshots/snapshot[namespace/complete='true' and position()=last()]" > \ + ${log} || true +} + +wait_for_snapshot_sync_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status) + local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status) + + mirror_image_snapshot "${cluster}" "${pool}" "${image}" + get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}" + local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + + get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}" + local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log}) + + test "${snapshot_id}" = "${primary_snapshot_id}" && return 0 + done + + return 1 + done + return 1 +} + +wait_for_replay_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then + wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image} + elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then + wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image} + else + return 1 + fi +} + + test_status_in_pool_dir() { local cluster=$1 @@ -700,7 +770,7 @@ test_status_in_pool_dir() local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status) CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} | - tee ${status_log} >&2 + tee ${status_log} >&2 grep "^ state: .*${state_pattern}" ${status_log} || return 1 grep "^ description: .*${description_pattern}" ${status_log} || return 1 @@ -756,8 +826,8 @@ wait_for_status_in_pool_dir() local service_pattern="$6" for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do - sleep ${s} - test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \ + sleep ${s} + test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \ "${description_pattern}" "${service_pattern}" && return 0 done @@ -772,12 +842,26 @@ create_image() local size=128 if [ -n "$1" ]; then - size=$1 - shift + size=$1 + shift fi rbd --cluster ${cluster} create --size ${size} \ - --image-feature layering,exclusive-lock,journaling $@ ${pool}/${image} + --image-feature layering,exclusive-lock,journaling $@ ${pool}/${image} +} + +create_image_and_enable_mirror() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local image=$1 ; shift + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + create_image ${cluster} ${pool} ${image} $@ + enable_mirror ${cluster} ${pool} ${image} ${mode} } enable_journaling() @@ -877,6 +961,25 @@ clone_image() --image-feature layering,exclusive-lock,journaling $@ } +clone_image_and_enable_mirror() +{ + local cluster=$1 + local parent_pool=$2 + local parent_image=$3 + local parent_snap=$4 + local clone_pool=$5 + local clone_image=$6 + shift 6 + + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@ + enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} +} + disconnect_image() { local cluster=$1 @@ -884,7 +987,7 @@ disconnect_image() local image=$3 rbd --cluster ${cluster} journal client disconnect \ - --image ${pool}/${image} + --image ${pool}/${image} } create_snapshot() @@ -957,7 +1060,7 @@ wait_for_snap_present() local s for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do - sleep ${s} + sleep ${s} rbd --cluster ${cluster} info ${pool}/${image}@${snap_name} || continue return 0 done @@ -980,7 +1083,7 @@ wait_for_snap_moved_to_trash() local s for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do - sleep ${s} + sleep ${s} test_snap_moved_to_trash $@ || continue return 0 done @@ -998,7 +1101,7 @@ wait_for_snap_removed_from_trash() local s for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do - sleep ${s} + sleep ${s} test_snap_removed_from_trash $@ || continue return 0 done @@ -1016,8 +1119,8 @@ write_image() test -n "${size}" || size=4096 rbd --cluster ${cluster} bench ${pool}/${image} --io-type write \ - --io-size ${size} --io-threads 1 --io-total $((size * count)) \ - --io-pattern rand + --io-size ${size} --io-threads 1 --io-total $((size * count)) \ + --io-pattern rand } stress_write_image() @@ -1028,9 +1131,9 @@ stress_write_image() local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}') timeout ${duration}s ceph_test_rbd_mirror_random_write \ - --cluster ${cluster} ${pool} ${image} \ - --debug-rbd=20 --debug-journaler=20 \ - 2> ${TEMPDIR}/rbd-mirror-random-write.log || true + --cluster ${cluster} ${pool} ${image} \ + --debug-rbd=20 --debug-journaler=20 \ + 2> ${TEMPDIR}/rbd-mirror-random-write.log || true } show_diff() @@ -1114,7 +1217,7 @@ set_pool_mirror_mode() { local cluster=$1 local pool=$2 - local mode=$3 + local mode=${3:-${MIRROR_POOL_MODE}} rbd --cluster=${cluster} mirror pool enable ${pool} ${mode} } @@ -1133,7 +1236,7 @@ enable_mirror() local cluster=$1 local pool=$2 local image=$3 - local mode=${4:-journal} + local mode=${4:-${MIRROR_IMAGE_MODE}} rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} } @@ -1170,8 +1273,8 @@ wait_for_image_present() # TODO: add a way to force rbd-mirror to update replayers for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do - sleep ${s} - test_image_present \ + sleep ${s} + test_image_present \ "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" && return 0 done @@ -1185,7 +1288,7 @@ get_image_id() local image=$3 rbd --cluster=${cluster} info ${pool}/${image} | - sed -ne 's/^.*block_name_prefix: rbd_data\.//p' + sed -ne 's/^.*block_name_prefix: rbd_data\.//p' } request_resync_image() diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh index efa00f4e18462..426edc42ff7ae 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -469,9 +469,9 @@ write_image ${CLUSTER2} ${POOL} ${image} 10 testlog " - replay stopped after disconnect" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" disconnect_image ${CLUSTER2} ${POOL} ${image} -test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' @@ -481,19 +481,19 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" compare_images ${POOL} ${image} testlog " - disconnected after max_concurrent_object_sets reached" if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_journal_max_concurrent_object_sets 1 write_image ${CLUSTER2} ${POOL} ${image} 20 16384 write_image ${CLUSTER2} ${POOL} ${image} 20 16384 - test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" + test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_journal_max_concurrent_object_sets 0 @@ -509,7 +509,7 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" compare_images ${POOL} ${image} testlog " - rbd_mirroring_resync_after_disconnect config option" @@ -522,13 +522,13 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" compare_images ${POOL} ${image} set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_mirroring_resync_after_disconnect false wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} disconnect_image ${CLUSTER2} ${POOL} ${image} -test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" +test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh new file mode 100755 index 0000000000000..7ab2239db03a3 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -0,0 +1,454 @@ +#!/bin/sh -ex +# +# rbd_mirror_snapshot.sh - test rbd-mirror daemon in snapshot-based mirroring mode +# +# The scripts starts two ("local" and "remote") clusters using mstart.sh script, +# creates a temporary directory, used for cluster configs, daemon logs, admin +# socket, temporary files, and launches rbd-mirror daemon. +# + +MIRROR_POOL_MODE=image +MIRROR_IMAGE_MODE=snapshot + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +testlog "TEST: add image and test replay" +start_mirrors ${CLUSTER1} +image=test +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" +set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' +fi +compare_images ${POOL} ${image} +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" + +testlog "TEST: stop mirror, add image, start mirror and test replay" +stop_mirrors ${CLUSTER1} +image1=test1 +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} +write_image ${CLUSTER2} ${POOL} ${image1} 100 +start_mirrors ${CLUSTER1} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' +fi +compare_images ${POOL} ${image1} + +testlog "TEST: test the first image is replaying after restart" +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: stop/start/restart mirror via admin socket" + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror start + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + flush ${CLUSTER1} + all_admin_daemons ${CLUSTER1} rbd mirror status +fi + +testlog "TEST: test image rename" +new_name="${image}_RENAMED" +rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} +mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} +admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} +mirror_image_snapshot ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: test trash move restore" +image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image}) +trash_move ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +trash_restore ${CLUSTER2} ${POOL} ${image_id} +enable_mirror ${CLUSTER2} ${POOL} ${image} snapshot +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: failover and failback" +start_mirrors ${CLUSTER2} + +# demote and promote same cluster +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +# failover (unmodified) +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} + +# failback (unmodified) +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +# failover +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} +write_image ${CLUSTER1} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +# failback +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +# force promote +force_promote_image=test_force_promote +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 + +testlog "TEST: cloned images" +testlog " - default" +parent_image=test_parent +parent_snap=snap +create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} +write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} + +clone_image=test_clone +clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image} +write_image ${CLUSTER2} ${POOL} ${clone_image} 100 +enable_mirror ${CLUSTER2} ${POOL} ${clone_image} snapshot + +wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' +compare_images ${PARENT_POOL} ${parent_image} + +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' +compare_images ${POOL} ${clone_image} + +testlog " - clone v1" +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}1 + +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v1 snapshot --rbd-default-clone-format 1 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 + +testlog " - clone v2" +parent_snap=snap_v2 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2 + +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2 +wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted' +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog " - clone v2 non-primary" +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog "TEST: data pool" +dp_image=test_data_pool +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' +compare_images ${POOL} ${dp_image}@snap1 +compare_images ${POOL} ${dp_image}@snap2 +compare_images ${POOL} ${dp_image} + +testlog "TEST: disable mirroring / delete non-primary image" +image2=test2 +image3=test3 +image4=test4 +image5=test5 +for i in ${image2} ${image3} ${image4} ${image5}; do + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i} + write_image ${CLUSTER2} ${POOL} ${i} 100 + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + fi + write_image ${CLUSTER2} ${POOL} ${i} 100 + mirror_image_snapshot ${CLUSTER2} ${POOL} ${i} + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' + wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' +done + +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +for i in ${image2} ${image4}; do + disable_mirror ${CLUSTER2} ${POOL} ${i} +done + +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2' +for i in ${image3} ${image5}; do + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + +for i in ${image2} ${image3} ${image4} ${image5}; do + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' +done + +testlog "TEST: snapshot rename" +snap_name='snap_rename' +enable_mirror ${CLUSTER2} ${POOL} ${image2} +create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0" +for i in `seq 1 20`; do + rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}" +done +mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2} +wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" + +testlog "TEST: disable mirror while daemon is stopped" +stop_mirrors ${CLUSTER1} +stop_mirrors ${CLUSTER2} +disable_mirror ${CLUSTER2} ${POOL} ${image} +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + test_image_present ${CLUSTER1} ${POOL} ${image} 'present' +fi +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: non-default namespace image mirroring" +testlog " - replay" +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 +write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' +compare_images ${POOL}/${NS1} ${image} +compare_images ${POOL}/${NS2} ${image} + +testlog " - disable mirroring / delete image" +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} +disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' +wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' + +testlog " - data pool" +dp_image=test_data_pool +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' +compare_images ${POOL}/${NS1} ${dp_image} + +testlog "TEST: simple image resync" +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: image resync while replayer is stopped" + admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + request_resync_image ${CLUSTER1} ${POOL} ${image} image_id + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + compare_images ${POOL} ${image} +fi + +testlog "TEST: request image resync while daemon is offline" +stop_mirrors ${CLUSTER1} +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +testlog "TEST: split-brain" +image=split-brain +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +promote_image ${CLUSTER1} ${POOL} ${image} --force +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${image} 10 +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + # teuthology will trash the daemon + testlog "TEST: no blacklists" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries" +fi diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index 7788863ffef2e..cb4f66b256c4f 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -97,7 +97,7 @@ for i in `seq 1 10` do stress_write_image ${CLUSTER2} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' snap_name="snap${i}" create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} -- 2.39.5