From 162a18998aa0da4076ff52a91a80646154c05b4e Mon Sep 17 00:00:00 2001 From: John Agombar Date: Thu, 30 Jan 2025 13:04:13 +0000 Subject: [PATCH] qa/workunits/rbd: updates to mirror group bash scripts - support cli parameters to specify the test to run - support cli parameter to specify the number of times to repeat the test - new tests - added RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR env variable in preparation for changes to group snapshot behaviour Signed-off-by: John Agombar --- qa/workunits/rbd/rbd_mirror_group.sh | 172 +- qa/workunits/rbd/rbd_mirror_group_simple.sh | 1945 ++++++++++++++++++- qa/workunits/rbd/rbd_mirror_helpers.sh | 638 +++++- 3 files changed, 2594 insertions(+), 161 deletions(-) mode change 100644 => 100755 qa/workunits/rbd/rbd_mirror_group_simple.sh diff --git a/qa/workunits/rbd/rbd_mirror_group.sh b/qa/workunits/rbd/rbd_mirror_group.sh index 9a1be4f03d8..e8f784fe972 100755 --- a/qa/workunits/rbd/rbd_mirror_group.sh +++ b/qa/workunits/rbd/rbd_mirror_group.sh @@ -7,7 +7,11 @@ # socket, temporary files, and launches rbd-mirror daemon. # -set -ex +if [ -n "${RBD_MIRROR_SHOW_CMD}" ]; then + set -e +else + set -ex +fi MIRROR_POOL_MODE=image MIRROR_IMAGE_MODE=snapshot @@ -45,7 +49,25 @@ fi testlog "TEST: add image to group and test replay" image=test-image create_image ${CLUSTER2} ${POOL} ${image} -group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + wait_for_group_not_present "${CLUSTER1}" "${POOL}" "${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" + wait_for_group_present "${CLUSTER2}" "${POOL}" "${group}" 1 +fi + wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' write_image ${CLUSTER2} ${POOL} ${image} 100 @@ -54,9 +76,43 @@ wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+replaying' 1 compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} testlog "TEST: test replay with remove image and later add the same" -group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 0 + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + wait_for_group_not_present "${CLUSTER1}" "${POOL}" "${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" + wait_for_group_present "${CLUSTER1}" "${POOL}" "${group}" 0 +fi + wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 0 -group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + wait_for_group_not_present "${CLUSTER1}" "${POOL}" "${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" +fi + wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 : ' # different pools - not MVP @@ -64,6 +120,10 @@ testlog "TEST: add image from a different pool to group and test replay" image0=test-image-diff-pool create_image ${CLUSTER2} ${PARENT_POOL} ${image0} group_image_add ${CLUSTER2} ${POOL}/${group} ${PARENT_POOL}/${image0} +if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 1 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" +fi wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 2 wait_for_image_present ${CLUSTER1} ${PARENT_POOL} ${image0} 'present' write_image ${CLUSTER2} ${PARENT_POOL} ${image0} 100 @@ -71,6 +131,10 @@ mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL} wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+replaying' compare_images ${PARENT_POOL} ${image0} group_image_remove ${CLUSTER1} ${POOL}/${group} ${PARENT_POOL}/${image0} +if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 2 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" +fi ' testlog "TEST: create regular group snapshots and test replay" @@ -93,7 +157,19 @@ group1=test-group1 create_group_and_enable_mirror ${CLUSTER2} ${POOL}/${group1} image1=test-image1 create_image ${CLUSTER2} ${POOL} ${image1} -group_image_add ${CLUSTER2} ${POOL}/${group1} ${POOL}/${image1} +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add ${CLUSTER2} ${POOL}/${group1} ${POOL}/${image1} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + mirror_group_snapshot "${CLUSTER2}" "${POOL}"/"${group1}" "${group_snap_id}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group1}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_add ${CLUSTER2} ${POOL}/${group1} ${POOL}/${image1} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group1}" +fi start_mirrors ${CLUSTER1} wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group1} 1 mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group1} @@ -132,15 +208,47 @@ fi testlog "TEST: add a large image to group and test replay" big_image=test-image-big create_image ${CLUSTER2} ${POOL} ${big_image} 1G -group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} write_image ${CLUSTER2} ${POOL} ${big_image} 1024 4194304 + +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 1 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_add ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" +fi + wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 2 -mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group} -test_group_and_image_sync_status ${CLUSTER1} ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} -group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} +mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group} +test_images_in_latest_synced_group ${CLUSTER1} ${POOL}/${group} 2 +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 1 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}"/"${group}" + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}"/"${group}" 'up+replaying' 1 + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_remove ${CLUSTER2} ${POOL}/${group} ${POOL}/${big_image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" +fi + remove_image_retry ${CLUSTER2} ${POOL} ${big_image} wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group} +test_images_in_latest_synced_group ${CLUSTER1} ${POOL}/${group} 1 testlog "TEST: test group rename" new_name="${group}_RENAMED" @@ -151,6 +259,9 @@ group_rename ${CLUSTER2} ${POOL}/${new_name} ${POOL}/${group} wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+replaying' 1 +# TODO add new image to syncing snapshot +# rollback needs to remove new image before rolling back +# also remove image and rollback testlog "TEST: failover and failback" start_mirrors ${CLUSTER2} @@ -200,7 +311,7 @@ wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 1 wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+replaying' 1 compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1} -testlog " - failback" +testlog " - failback to cluster2" mirror_group_demote ${CLUSTER1} ${POOL}/${group1} wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group1} wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 0 @@ -213,7 +324,7 @@ wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+replaying' 1 wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+stopped' 1 compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1} -testlog " - force promote" +testlog " - force promote cluster1" wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 write_image ${CLUSTER2} ${POOL} ${image} 100 mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group} @@ -261,8 +372,26 @@ create_group_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1}/${group} create_group_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2}/${group} create_image ${CLUSTER2} ${POOL}/${NS1} ${image} create_image ${CLUSTER2} ${POOL}/${NS2} ${image} -group_image_add ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} -group_image_add ${CLUSTER2} ${POOL}/${NS2}/${group} ${POOL}/${NS2}/${image} + +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} + group_image_add ${CLUSTER2} ${POOL}/${NS2}/${group} ${POOL}/${NS2}/${image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}/${NS1}/${group}" 'up+replaying' 0 + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}/${NS2}/${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}/${NS1}/${group}" + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}/${NS2}/${group}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_add ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} + group_image_add ${CLUSTER2} ${POOL}/${NS2}/${group} ${POOL}/${NS2}/${image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" +fi + wait_for_group_replay_started ${CLUSTER1} ${POOL}/${NS1}/${group} 1 wait_for_group_replay_started ${CLUSTER1} ${POOL}/${NS2}/${group} 1 write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 @@ -275,7 +404,21 @@ compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${image} compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${POOL}/${NS2} ${image} testlog " - disable mirroring / remove group" -group_image_remove ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} +if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${CLUSTER1}" "${POOL}/${NS1}/${group}" 'up+replaying' 1 + mirror_group_snapshot_and_wait_for_sync_complete "${CLUSTER1}" "${CLUSTER2}" "${POOL}/${NS1}/${group}" + fi +else + mirror_group_disable "${CLUSTER2}" "${POOL}/${group}" + echo "temp workaround - sleep 5" # TODO remove + sleep 5 + group_image_remove ${CLUSTER2} ${POOL}/${NS1}/${group} ${POOL}/${NS1}/${image} + mirror_group_enable "${CLUSTER2}" "${POOL}/${group}" +fi + remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' group_remove ${CLUSTER1} ${POOL}/${NS1}/${group} @@ -289,6 +432,7 @@ image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) wait_for_image_present ${CLUSTER2} ${POOL} ${image} 'present' wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' mirror_group_resync ${CLUSTER1} ${POOL}/${group} +# original image is deleted and recreated wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1 diff --git a/qa/workunits/rbd/rbd_mirror_group_simple.sh b/qa/workunits/rbd/rbd_mirror_group_simple.sh old mode 100644 new mode 100755 index 6eff3b244bc..d6931eb7ac7 --- a/qa/workunits/rbd/rbd_mirror_group_simple.sh +++ b/qa/workunits/rbd/rbd_mirror_group_simple.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash + #!/usr/bin/env bash # # rbd_mirror_group_simple.sh # @@ -6,41 +6,98 @@ # It may repeat some of the tests from rbd_mirror_group.sh, but only those that are known to work # It has a number of extra tests that imclude multiple images in a group # +# shellcheck disable=SC2034 # Don't warn about unused variables and functions +# shellcheck disable=SC2317 # Don't warn about unreachable commands + export RBD_MIRROR_NOCLEANUP=1 export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror export RBD_MIRROR_SHOW_CMD=1 export RBD_MIRROR_MODE=snapshot +group0=test-group0 +group1=test-group1 +pool0=mirror +pool1=mirror_parent +image_prefix=test-image + +feature=${4:-0} +features=( + "layering,exclusive-lock,object-map,fast-diff,deep-flatten" + "layering,exclusive-lock,object-map,fast-diff" + "layering,exclusive-lock" + "layering" +) + +# save and clear the cli args (can't call rbd_mirror_helpers with these defined) +args=("$@") +set -- + + +if [ -z "${RBD_IMAGE_FEATURES}" ]; then + echo "RBD_IMAGE_FEATURES=${features[${feature}]}" + RBD_IMAGE_FEATURES=${features[${feature}]} +fi + . $(dirname $0)/rbd_mirror_helpers.sh +# create group with images then enable mirroring. Remove group without disabling mirroring +declare -a test_create_group_with_images_then_mirror_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false' 5) +# create group with images then enable mirroring. Disable mirroring then remove group +declare -a test_create_group_with_images_then_mirror_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' 5) +declare -a test_create_group_with_images_then_mirror_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' 30) + +test_create_group_with_images_then_mirror_scenarios=3 + test_create_group_with_images_then_mirror() { - local primary_cluster=$1 - local secondary_cluster=$2 - local pool=$3 - local group=$4 - local image_prefix=$5 - local disable_before_remove=$6 + local primary_cluster=$1 ; shift + local secondary_cluster=$1 ; shift + local pool=$1 ; shift + local group=$1 ; shift + local image_prefix=$1 ; shift + local disable_before_remove=$1 ; shift + local image_count=$1 ; shift + if [ -n "$1" ]; then + local get_times='true' + local -n _times_result_arr=$8 + fi group_create "${primary_cluster}" "${pool}/${group}" - images_create "${primary_cluster}" "${pool}/${image_prefix}" 5 - group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" 5 + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${image_count}" + # write to every image in the group + local io_count=10240 + local io_size=4096 + for loop_instance in $(seq 0 $(("${image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + + local start_time enabled_time synced_time + start_time=$(date +%s) mirror_group_enable "${primary_cluster}" "${pool}/${group}" + enabled_time=$(date +%s) # rbd group list poolName (check groupname appears in output list) # do this before checking for replay_started because queries directed at the daemon fail with an unhelpful # error message before the group appears on the remote cluster - wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 5 + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${image_count}" check_daemon_running "${secondary_cluster}" # ceph --daemon mirror group status groupName - wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 5 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${image_count}" check_daemon_running "${secondary_cluster}" # rbd mirror group status groupName - wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 5 + #sleep 10 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${image_count}" + wait_for_group_synced "${primary_cluster}" "${pool}/${group}" + synced_time=$(date +%s) + if [ -n "$get_times" ]; then + _times_result_arr+=($((enabled_time-start_time))) + _times_result_arr+=($((synced_time-enabled_time))) + fi check_daemon_running "${secondary_cluster}" if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then @@ -49,7 +106,7 @@ test_create_group_with_images_then_mirror() check_daemon_running "${secondary_cluster}" if [ 'false' != "${disable_before_remove}" ]; then - mirror_group_disable "${primary_cluster}" "${pool}/${group}" + mirror_group_disable "${primary_cluster}" "${pool}/${group}" fi group_remove "${primary_cluster}" "${pool}/${group}" @@ -60,9 +117,327 @@ test_create_group_with_images_then_mirror() wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" check_daemon_running "${secondary_cluster}" - images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5 + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" +} + +# record the time taken to enable and sync for a group with increasing number of images. +declare -a test_group_enable_times_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}") + +test_group_enable_times_scenarios=1 + +test_group_enable_times() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local disable_before_remove='true' + local image_count + local results=() + local times=() + + for image_count in {0,5,10,15,20,25,30}; do + times=() + test_create_group_with_images_then_mirror "${primary_cluster}" "${secondary_cluster}" "${pool}" "${group}" "${image_prefix}" 'true' "${image_count}" times + results+=("image count:$image_count enable time:"${times[0]}" sync_time:"${times[1]}) + done + + for result in "${results[@]}"; do + echo -e "${RED}${result}${NO_COLOUR}" + done + +#results: +#image count:0 enable time:0 sync_time:6 +#image count:5 enable time:4 sync_time:6 +#image count:10 enable time:9 sync_time:9 +#image count:15 enable time:15 sync_time:13 +#image count:20 enable time:20 sync_time:13 +#image count:25 enable time:25 sync_time:21 +#image count:30 enable time:30 sync_time:22 + +} + +# create group with images then enable mirroring. Remove group without disabling mirroring +declare -a test_create_group_with_image_remove_then_repeat_1=("${CLUSTER2}" "${CLUSTER1}" ) + +test_create_group_with_image_remove_then_repeat_scenarios=1 + +test_create_group_with_image_remove_then_repeat() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool="${pool0}" + local group="${group0}" + + images_create "${primary_cluster}" "${pool}/${image_prefix}" 1 + group_create "${primary_cluster}" "${pool}/${group}" + + local loop_instance + for loop_instance in $(seq 0 5); do + testlog "test_create_group_with_image_remove_then_repeat ${loop_instance}" + + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}0" + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 1 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 1 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 1 + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}0" + + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 0 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0 + + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + check_daemon_running "${secondary_cluster}" + done + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" 1 +} + +# create group with images then enable mirroring. Disable and re-enable repeatedly +declare -a test_enable_disable_repeat_1=("${CLUSTER2}" "${CLUSTER1}" ) + +test_enable_disable_repeat_scenarios=1 + +test_enable_disable_repeat() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool="${pool0}" + local group="group" + + local group_instance + for group_instance in $(seq 0 9); do + group_create "${primary_cluster}" "${pool}/${group}${group_instance}" + done + + stop_mirrors "${primary_cluster}" + stop_mirrors "${secondary_cluster}" + start_mirrors "${secondary_cluster}" + + local loop_instance + for loop_instance in $(seq 0 10); do + testlog "test_enable_disable_repeat ${loop_instance}" + + for group_instance in $(seq 0 9); do + mirror_group_enable "${primary_cluster}" "${pool}/${group}${group_instance}" + done + + for group_instance in $(seq 0 9); do + mirror_group_disable "${primary_cluster}" "${pool}/${group}${group_instance}" + done + done + + check_daemon_running "${secondary_cluster}" + + for group_instance in $(seq 0 9); do + group_remove "${primary_cluster}" "${pool}/${group}${group_instance}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}${group_instance}" + done +} + +# add and remove images to/from a mirrored group +declare -a test_mirrored_group_add_and_remove_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 5) + +test_mirrored_group_add_and_remove_images_scenarios=1 + +test_mirrored_group_add_and_remove_images() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local group_image_count=$6 + + group_create "${primary_cluster}" "${pool}/${group}" + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0 + + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + # create another image and populate it with some data + local image_name="test_image" + image_create "${primary_cluster}" "${pool}/${image_name}" + local io_count=10240 + local io_size=4096 + write_image "${primary_cluster}" "${pool}" "${image_name}" "${io_count}" "${io_size}" + + # add, wait for stable and then remove the image from the group + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $((1+"${group_image_count}")) + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $((1+"${group_image_count}")) + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + fi + + # re-add and immediately remove + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + + # check that expected number of images exist on secondary + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + # remove and immediately re-add a different image + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}2" + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}2" + + # check that expected number of images exist on secondary + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + # remove all images from the group + group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + image_remove "${primary_cluster}" "${pool}/${image_name}" + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + # check that expected number of images exist on secondary - TODO this should be replaying, but deleting the last image seems to cause + # the group to go stopped atm + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0 + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + check_daemon_running "${secondary_cluster}" +} + +# create group with images then enable mirroring. Remove all images from group and check state matches initial empty group state +declare -a test_mirrored_group_remove_all_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 2) + +test_mirrored_group_remove_all_images_scenarios=1 + +test_mirrored_group_remove_all_images() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local group_image_count=$6 + + group_create "${primary_cluster}" "${pool}/${group}" + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 0 + + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + + # remove all images from the group + group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + # check that expected number of images exist on secondary + # TODO why is the state "stopped" - a new empty group is in the "replaying" state + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0 + + # adding the images back into the group causes it to go back to replaying + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + # remove all images from the group again + group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + + # check that expected number of images exist on secondary + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0 + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + check_daemon_running "${secondary_cluster}" } +# create group then enable mirroring before adding images to the group. Disable mirroring before removing group +declare -a test_create_group_mirror_then_add_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false') +# create group then enable mirroring before adding images to the group. Remove group with mirroring enabled +declare -a test_create_group_mirror_then_add_images_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true') + +test_create_group_mirror_then_add_images_scenarios=2 + test_create_group_mirror_then_add_images() { local primary_cluster=$1 @@ -85,6 +460,12 @@ test_create_group_mirror_then_add_images() images_create "${primary_cluster}" "${pool}/${image_prefix}" 5 group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" 5 + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + fi + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 5 check_daemon_running "${secondary_cluster}" @@ -100,7 +481,7 @@ test_create_group_mirror_then_add_images() check_daemon_running "${secondary_cluster}" if [ 'false' != "${disable_before_remove}" ]; then - mirror_group_disable "${primary_cluster}" "${pool}/${group}" + mirror_group_disable "${primary_cluster}" "${pool}/${group}" fi group_remove "${primary_cluster}" "${pool}/${group}" @@ -114,6 +495,118 @@ test_create_group_mirror_then_add_images() images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5 } +#test remote namespace with different name +declare -a test_remote_namespace_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}") + +test_remote_namespace_scenarios=1 + +test_remote_namespace() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + + # configure primary namespace mirrored to secondary namespace with a different name + local primary_namespace + primary_namespace='ns3' + local secondary_namespace + secondary_namespace='ns3_remote' + + local primary_pool_spec + local secondary_pool_spec + primary_pool_spec="${pool}/${primary_namespace}" + secondary_pool_spec="${pool}/${secondary_namespace}" + + run_cmd "rbd --cluster ${primary_cluster} namespace create ${pool}/${primary_namespace}" + run_cmd "rbd --cluster ${secondary_cluster} namespace create ${pool}/${secondary_namespace}" + + run_cmd "rbd --cluster ${primary_cluster} mirror pool enable ${pool}/${primary_namespace} image --remote-namespace ${secondary_namespace}" + run_cmd "rbd --cluster ${secondary_cluster} mirror pool enable ${pool}/${secondary_namespace} image --remote-namespace ${primary_namespace}" + + group_create "${primary_cluster}" "${primary_pool_spec}/${group}" + mirror_group_enable "${primary_cluster}" "${primary_pool_spec}/${group}" + wait_for_group_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" 0 + wait_for_group_replay_started "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 0 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 0 + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0 + fi + + try_cmd "rbd --cluster ${secondary_cluster} group snap list ${secondary_pool_spec}/${group}" || : + try_cmd "rbd --cluster ${primary_cluster} group snap list ${primary_pool_spec}/${group}" || : + + mirror_group_disable "${primary_cluster}" "${primary_pool_spec}/${group}" + + try_cmd "rbd --cluster ${secondary_cluster} group snap list ${secondary_pool_spec}/${group}" || : + try_cmd "rbd --cluster ${primary_cluster} group snap list ${primary_pool_spec}/${group}" || : + + group_remove "${primary_cluster}" "${primary_pool_spec}/${group}" + wait_for_group_not_present "${primary_cluster}" "${primary_pool_spec}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" + check_daemon_running "${secondary_cluster}" + + # repeat the test - this time with some images + group_create "${primary_cluster}" "${primary_pool_spec}/${group}" + images_create "${primary_cluster}" "${primary_pool_spec}/${image_prefix}" 5 + group_images_add "${primary_cluster}" "${primary_pool_spec}/${group}" "${primary_pool_spec}/${image_prefix}" 5 + + mirror_group_enable "${primary_cluster}" "${primary_pool_spec}/${group}" + wait_for_group_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" 5 + wait_for_group_replay_started "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 5 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 5 + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0 + fi + + # try a manual snapshot and check that it syncs + write_image "${primary_cluster}" "${primary_pool_spec}" "${image_prefix}0" 10 4096 + local group_snap_id + mirror_group_snapshot "${primary_cluster}" "${primary_pool_spec}/${group}" group_snap_id + wait_for_group_snap_present "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}" + wait_for_group_snap_sync_complete "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}" + + # Check all images in the group and confirms that they are synced + test_group_synced_image_status "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}" 5 + + # try demote, promote and resync + mirror_group_demote "${primary_cluster}" "${primary_pool_spec}/${group}" + wait_for_group_replay_stopped "${secondary_cluster}" "${secondary_pool_spec}/${group}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}/${group}" 'up+stopped' 0 + wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}/${group}" 'down+unknown' 0 + mirror_group_promote "${secondary_cluster}" "${secondary_pool_spec}/${group}" + + write_image "${secondary_cluster}" "${secondary_pool_spec}" "${image_prefix}0" 10 4096 + + mirror_group_demote "${secondary_cluster}" "${secondary_pool_spec}/${group}" + mirror_group_promote "${primary_cluster}" "${primary_pool_spec}/${group}" + + mirror_group_resync "${secondary_cluster}" "${secondary_pool_spec}/${group}" + + wait_for_group_synced "${primary_cluster}" "${primary_pool_spec}/${group}" + + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 5 + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0 + fi + + group_remove "${primary_cluster}" "${primary_pool_spec}/${group}" + wait_for_group_not_present "${primary_cluster}" "${primary_pool_spec}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" + + images_remove "${primary_cluster}" "${primary_pool_spec}/${image_prefix}" 5 + check_daemon_running "${secondary_cluster}" +} + +#test empty group +declare -a test_empty_group_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}") +#test empty group with namespace +declare -a test_empty_group_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}/${NS1}" "${group0}") + +test_empty_group_scenarios=2 + test_empty_group() { local primary_cluster=$1 @@ -155,6 +648,92 @@ test_empty_group() check_daemon_running "${secondary_cluster}" } +#check that omap keys have been correctly deleted +declare -a test_empty_group_omap_keys_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}") + +test_empty_group_omap_keys_scenarios=1 + +test_empty_group_omap_keys() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + + wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gremote_status" + wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global" + wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gremote_status" + wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global" + + run_test test_empty_group 1 + + wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gremote_status" + wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global" + wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gremote_status" + wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global" +} + +#check that pool doesn't get in strange state +declare -a test_group_with_clone_image_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}") + +test_group_with_clone_image_scenarios=1 + +test_group_with_clone_image() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + + group_create "${primary_cluster}" "${pool}/${group}" + + # create an image and then clone it + image_create "${primary_cluster}" "${pool}/parent_image" + create_snapshot "${primary_cluster}" "${pool}" "parent_image" "snap1" + protect_snapshot "${primary_cluster}" "${pool}" "parent_image" "snap1" + clone_image "${primary_cluster}" "${pool}" "parent_image" "snap1" "${pool}" "child_image" + + # create some other images + image_create "${primary_cluster}" "${pool}/other_image0" + image_create "${primary_cluster}" "${pool}/other_image1" + + # add the other images and the clone to the group + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/other_image0" + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/other_image1" + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/child_image" + + # next command fails with the following message + # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: image_enable: mirroring is not enabled for the parent + # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: group_enable: failed enabling image: child_image: (22) Invalid argument + mirror_group_enable_try "${primary_cluster}" "${pool}/${group}" || : + test 0 = "$(grep -c "interrupted" "$CMD_STDERR")" || fail "unexpected output" + + # next command appears to succeed + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + + # another attempt at enable fails with a strange message + # 2025-01-30T17:17:34.421+0000 7f9ad0d74b40 -1 librbd::api::Mirror: group_enable: enabling mirroring for group test-group0 either in progress or was interrupted + mirror_group_enable_try "${primary_cluster}" "${pool}/${group}" || : + test 0 = "$(grep -c "interrupted" "$CMD_STDERR")" || fail "unexpected output" +} + +test_from_nithya_that_will_stop_working_when_api_changes() +{ +[root@server1 build]# rbd-a group create data/grp1 +[root@server1 build]# rbd-a group image add data/grp1 data/img-1 +[root@server1 build]# rbd-a group image add data/grp1 data/img-2 +[root@server1 build]# rbd-a group image add data/grp1 data/img-3 +[root@server1 build]# rbd-a mirror group enable data/grp1 +[root@server1 build]# rbd-a mirror image demote data/img-2 +[root@server1 build]# rbd-a mirror group snapshot data/grp1 +[root@server1 build]# rbd-a snap ls --all data/img-3 +[root@server1 build]# rbd-a group snap ls data/grp1 +} + +# test two empty groups +declare -a test_empty_groups_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${group1}") + +test_empty_groups_scenarios=1 + test_empty_groups() { local primary_cluster=$1 @@ -198,6 +777,11 @@ test_empty_groups() check_daemon_running "${secondary_cluster}" } +# add image from a different pool to group and test replay +declare -a test_images_different_pools_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${pool1}" "${group0}" "${image_prefix}") + +test_images_different_pools_scenarios=1 + # This test is not MVP test_images_different_pools() { @@ -223,6 +807,12 @@ test_images_different_pools() image_create "${primary_cluster}" "${pool1}/${image_prefix}1" group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool1}/${image_prefix}1" + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + # check secondary cluster sees 0 images + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 0 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}"/"${group}" + fi + wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 2 wait_for_group_replay_started "${secondary_cluster}" "${pool0}"/"${group}" 2 wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 2 @@ -240,6 +830,11 @@ test_images_different_pools() image_remove "${primary_cluster}" "${pool1}/${image_prefix}1" } +# create regular group snapshots and test replay +declare -a test_create_group_with_images_then_mirror_with_regular_snapshots_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}") + +test_create_group_with_images_then_mirror_with_regular_snapshots_scenarios=1 + test_create_group_with_images_then_mirror_with_regular_snapshots() { local primary_cluster=$1 @@ -247,6 +842,7 @@ test_create_group_with_images_then_mirror_with_regular_snapshots() local pool=$3 local group=$4 local image_prefix=$5 + local snap='regular_snap' group_create "${primary_cluster}" "${pool}/${group}" images_create "${primary_cluster}" "${pool}/${image_prefix}" 5 @@ -261,14 +857,15 @@ test_create_group_with_images_then_mirror_with_regular_snapshots() wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 fi - snap='regular_snap' check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}" check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}" group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}" check_group_snap_exists "${primary_cluster}" "${pool}/${group}" "${snap}" # snap is currently copied to secondary cluster, where it remains in the "incomplete" state, but this is maybe incorrect - see slack thread TODO + # - should not be copied until mirrored. mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}" group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}" @@ -296,8 +893,65 @@ test_create_group_with_images_then_mirror_with_regular_snapshots() images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5 } -test_create_group_with_large_image() -{ +# create regular group snapshots before enable mirroring +declare -a test_create_group_with_regular_snapshots_then_mirror_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}") + +test_create_group_with_regular_snapshots_then_mirror_scenarios=1 + +test_create_group_with_regular_snapshots_then_mirror() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local group_image_count=12 + local snap='regular_snap' + + group_create "${primary_cluster}" "${pool}/${group}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}" + check_group_snap_exists "${primary_cluster}" "${pool}/${group}" "${snap}" + + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}" +# wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}" +# wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + #if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then +# wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 +# fi + + check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}" + # TODO this next command fails because the regular snapshot seems to get stuck in the "incomplete" state on the secondary + # and the mirror group snapshot (taken on mirror enable) never appears on the secondary. + wait_for_group_synced "${primary_cluster}" "${pool}/${group}" +## mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + + group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}" + check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}" + # this next extra mirror_group_snapshot should not be needed - waiting for fix TODO + mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}" + + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" +} + +# add a large image to group and test replay +declare -a test_create_group_with_large_image_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}") + +test_create_group_with_large_image_scenarios=1 + +test_create_group_with_large_image() +{ local primary_cluster=$1 local secondary_cluster=$2 local pool0=$3 @@ -319,19 +973,53 @@ test_create_group_with_large_image() big_image=test-image-big image_create "${primary_cluster}" "${pool0}/${big_image}" 4G - group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" - write_image "${primary_cluster}" "${pool0}" "${big_image}" 1024 4194304 - wait_for_group_replay_started "${secondary_cluster}" "${pool0}/${group}" 2 - mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}" + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool0}/${group}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}/${group}" 'up+replaying' 2 + fi + else + mirror_group_disable "${primary_cluster}" "${pool0}/${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool0}" "${group}" + group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" + mirror_group_enable "${primary_cluster}" "${pool0}/${group}" + wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 2 + fi - test_group_and_image_sync_status "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" + write_image "${primary_cluster}" "${pool0}" "${big_image}" 1024 4194304 + local group_snap_id + mirror_group_snapshot "${primary_cluster}" "${pool0}/${group}" group_snap_id + wait_for_group_snap_present "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}" + + # TODO if the sync process could be controlled then we could check that test-image is synced before test-image-big + # and that the group is only marked as synced once both images have completed their sync + wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}" + + # Check all images in the group and confirms that they are synced + test_group_synced_image_status "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}" 2 + + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 1 + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}"/"${group}" + fi + else + mirror_group_disable "${primary_cluster}" "${pool0}/${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool0}" "${group}" + group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" + mirror_group_enable "${primary_cluster}" "${pool0}/${group}" + wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 1 + fi - group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}" remove_image_retry "${primary_cluster}" "${pool0}" "${big_image}" - wait_for_group_replay_started "${secondary_cluster}" "${pool0}/${group}" 1 mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}" + test_images_in_latest_synced_group "${secondary_cluster}" "${pool0}/${group}" 1 mirror_group_disable "${primary_cluster}" "${pool0}/${group}" group_remove "${primary_cluster}" "${pool0}/${group}" @@ -341,6 +1029,11 @@ test_create_group_with_large_image() image_remove "${primary_cluster}" "${pool0}/${image_prefix}" } +# multiple images in group with io +declare -a test_create_group_with_multiple_images_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}") + +test_create_group_with_multiple_images_do_io_scenarios=1 + test_create_group_with_multiple_images_do_io() { local primary_cluster=$1 @@ -365,14 +1058,22 @@ test_create_group_with_multiple_images_do_io() local io_count=1024 local io_size=4096 - write_image "${primary_cluster}" "${pool}" "${image_prefix}0" "${io_count}" "${io_size}" + + local loop_instance + for loop_instance in $(seq 0 $((5-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}" + test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" 5 -exit 0 - # TODO this test needs finishing. The next function is not yet complete - see the TODO in it - test_group_and_image_sync_status "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}" "${pool1}/${big_image}" + for loop_instance in $(seq 0 $((5-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}" + done + for loop_instance in $(seq 0 $((5-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done snap='regular_snap' group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}" @@ -380,6 +1081,22 @@ exit 0 mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}" + test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" 5 + + for loop_instance in $(seq 0 $((5-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}" + done + + group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}" + check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}" + # this next extra mirror_group_snapshot should not be needed - waiting for fix TODO + mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}" + check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}" + + for loop_instance in $(seq 0 $((5-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}" + done mirror_group_disable "${primary_cluster}" "${pool}/${group}" group_remove "${primary_cluster}" "${pool}/${group}" @@ -389,68 +1106,1146 @@ exit 0 images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5 } -set -ex +# multiple images in group with io +declare -a test_stopped_daemon_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 3) -# If the tmpdir or cluster conf file doesn't exist then assume that the cluster needs setting up -if [ ! -d "${RBD_MIRROR_TEMDIR}" ] || [ ! -f "${RBD_MIRROR_TEMDIR}"'/cluster1.conf' ] -then - setup -fi -export RBD_MIRROR_USE_EXISTING_CLUSTER=1 +test_stopped_daemon_scenarios=1 + +test_stopped_daemon() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local group_image_count=$6 -# rbd_mirror_helpers assumes that we are running from tmpdir -setup_tempdir + check_daemon_running "${secondary_cluster}" -# see if we need to (re)start rbd-mirror deamon -pid=$(cat "$(daemon_pid_file "${CLUSTER1}")" 2>/dev/null) || : -if [ -z "${pid}" ] -then - start_mirrors "${CLUSTER1}" -fi -check_daemon_running "${CLUSTER1}" 'restart' + group_create "${primary_cluster}" "${pool}/${group}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" -group0=test-group0 -group1=test-group1 -pool0=mirror -pool1=mirror_parent -image_prefix=test-image + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}" + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}" + + local primary_group_snap_id + get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id + local secondary_group_snap_id + get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id + test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; } + + # Add image to synced group (whilst daemon is stopped) + echo "stopping daemon" + stop_mirrors "${secondary_cluster}" + + local image_name="test_image" + image_create "${primary_cluster}" "${pool}/${image_name}" + + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool}/${group}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}/${group}" 'up+replaying' $(("${group_image_count}"+1)) + fi + else + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + # wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + # wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" $(("${group_image_count}"+1)) + fi + + get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id + test "${primary_group_snap_id}" != "${secondary_group_snap_id}" || { fail "matched ids"; return 1; } + + echo "starting daemon" + start_mirrors "${secondary_cluster}" + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" $(("${group_image_count}"+1)) + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $(("${group_image_count}"+1)) + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}" + + get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id + test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; } + + # removed image from synced group (whilst daemon is stopped) + echo "stopping daemon" + stop_mirrors "${secondary_cluster}" + + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + + if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then + mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool}/${group}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}/${group}" 'up+replaying' $(("${group_image_count}")) + fi + else + mirror_group_disable "${primary_cluster}" "${pool}/${group}" +# wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}" + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + # wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" $(("${group_image_count}")) + fi + + get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id + test "${primary_group_snap_id}" != "${secondary_group_snap_id}" || { fail "matched ids"; return 1; } + + echo "starting daemon" + start_mirrors "${secondary_cluster}" + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + # TODO next command fails because rbd group snap list command fails with -2 + # though group does exist on secondary + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}" + + get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id + test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; } + + # TODO test more actions whilst daemon is stopped + # add image, take snapshot, remove image, take snapshot, restart + # disable mirroring + + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" +} + +# multiple images in group and standalone images too with io +declare -a test_group_and_standalone_images_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false') +declare -a test_group_and_standalone_images_do_io_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true') + +# TODO scenario 2 fails currently +test_group_and_standalone_images_do_io_scenarios=1 + +test_group_and_standalone_images_do_io() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group=$4 + local image_prefix=$5 + local test_pool_status=$6 + + local standalone_image_prefix=standalone-image + local standalone_image_count=4 + local group_image_count=2 + + images_create "${primary_cluster}" "${pool}/${standalone_image_prefix}" "${standalone_image_count}" + + group_create "${primary_cluster}" "${pool}/${group}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}" + + if [ 'true' = "${test_pool_status}" ]; then + local fields=(//status/images/image/name //status/groups/group/name) + local pool_fields_count_arr=() + count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}" + # Check count of images and groups in the command output + echo "kk ${pool_fields_count_arr[0]} ${pool_fields_count_arr[1]}" + test 0 = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}" + test 0 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}" + fi + + mirror_group_enable "${primary_cluster}" "${pool}/${group}" + + if [ 'true' = "${test_pool_status}" ]; then + local fields=(//status/images/image/name //status/groups/group/name) + pool_fields_count_arr=() + count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}" + # Check count of images and groups in the command output + test $((${group_image_count})) = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}" + test 1 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}" + fi + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}" + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0 + fi + + # enable mirroring for standalone images + local loop_instance + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + enable_mirror "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + wait_for_image_replay_started "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + wait_for_replay_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + wait_for_replaying_status_in_pool_dir "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + compare_images "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + done + + if [ 'true' = "${test_pool_status}" ]; then + local fields=(//status/images/image/name //status/groups/group/name) + pool_fields_count_arr=() + count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}" + # Check count of images and groups in the command output + test $((${standalone_image_count}+${group_image_count})) = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}" + test 1 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}" + fi + + local io_count=1024 + local io_size=4096 + + # write to all of the images + for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + + # snapshot the group + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}" + test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" "${group_image_count}" + + for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}" + done + + # snapshot the individual images too, wait for sync and compare + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + mirror_image_snapshot "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + wait_for_snapshot_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + done + + # do more IO + for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + + # Snapshot the group and images. Sync both in parallel + local group_snap_id + mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" group_snap_id + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + mirror_image_snapshot "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + done + + wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}" + for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}" + done + + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + wait_for_snapshot_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + done + + mirror_group_disable "${primary_cluster}" "${pool}/${group}" + group_remove "${primary_cluster}" "${pool}/${group}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}" + + # re-check images + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}" + done + + # disable mirroring for standalone images + local loop_instance + for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do + disable_mirror "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" + wait_for_image_present "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" 'deleted' + done + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}" + images_remove "${primary_cluster}" "${pool}/${standalone_image_prefix}" "${standalone_image_count}" +} + +# multiple groups with images in each with io +# mismatched size groups (same size images) +declare -a test_create_multiple_groups_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 128 5 2 128) +declare -a test_create_multiple_groups_do_io_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 128 1 1 128) +# mismatched size groups (mismatched size images) +declare -a test_create_multiple_groups_do_io_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 4 5 2 128) +declare -a test_create_multiple_groups_do_io_4=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 4 1 1 128) +# equal size groups +declare -a test_create_multiple_groups_do_io_5=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 2 5 128 2 5 128) + +test_create_multiple_groups_do_io_scenarios=5 + +test_create_multiple_groups_do_io() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local group_a_count=$4 + local group_a_image_count=$5 + local group_a_image_size=$6 + local group_b_count=$7 + local group_b_image_count=$8 + local group_b_image_size=$9 + + local image_count=$(("${group_a_count}"*"${group_a_image_count}" + "${group_b_count}"*"${group_b_image_count}")) + local image_prefix='test-image' + local group_prefix='test-group' + local loop_instance + local group_instance + + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + group_create "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + images_create "${primary_cluster}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}" "${group_a_image_size}" + group_images_add "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + group_create "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + images_create "${primary_cluster}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}" "${group_b_image_size}" + group_images_add "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}" + done + + # enable mirroring for every group + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + mirror_group_enable "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + mirror_group_enable "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + done + + # check that every group appears on the secondary + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + wait_for_group_present "${secondary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}" "${group_a_image_count}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + wait_for_group_present "${secondary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}" "${group_b_image_count}" + done + + # export RBD_MIRROR_INSTANCES=X and rerun the test to check assignment + # image size appears not to influence the distribution of group replayers + # number of images in a group does influence the distribution + # FUTURE - implement some checking on the assignment rather than just printing it + # - also maybe change image count in groups and check rebalancing + local group_arr + local image_arr + for instance in $(seq 0 ${LAST_MIRROR_INSTANCE}); do + local result + query_replayer_assignment "${secondary_cluster}" "${instance}" result + group_arr+=("${result[0]}") + image_arr+=("${result[1]}") + group_count_arr+=("${result[2]}") + image_count_arr+=("${result[3]}") + done + for instance in $(seq 0 ${LAST_MIRROR_INSTANCE}); do + echo -e "${RED}MIRROR DAEMON INSTANCE:${instance}${NO_COLOUR}"; + echo -e "${RED}GROUP_REPLAYERS:${group_count_arr[$instance]}${NO_COLOUR}"; + echo -e "${RED}${group_arr[$instance]}${NO_COLOUR}"; + echo -e "${RED}IMAGE_REPLAYERS:${image_count_arr[$instance]}${NO_COLOUR}"; + echo -e "${RED}${image_arr[$instance]}${NO_COLOUR}"; + done + + # check that every group and image are in the correct state on the secondary + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" "${group_a_image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" 'up+replaying' "${group_a_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" 'down+unknown' 0 + fi + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" "${group_b_image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" 'up+replaying' "${group_b_image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" 'down+unknown' 0 + fi + done + + local io_count=10240 + local io_size=4096 + local group_to_mirror='-a0' + + # write to every image in one a group, mirror group and compare images + for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" "${io_count}" "${io_size}" + done + + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}${group_to_mirror}" + + for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" + done + + group_to_mirror='-b0' + + # write to every image in one b group, mirror group and compare images + for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" "${io_count}" "${io_size}" + done + + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}${group_to_mirror}" + + for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" + done + + # write to one image in every group, mirror groups and compare images + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}-a${loop_instance}-0" "${io_count}" "${io_size}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}-b${loop_instance}-0" "${io_count}" "${io_size}" + done + + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + done + + for group_instance in $(seq 0 $(("${group_a_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}" + done + done + for group_instance in $(seq 0 $(("${group_b_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}" + done + done + + # write to every image in every group, mirror groups and compare images + for group_instance in $(seq 0 $(("${group_a_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}" "${io_count}" "${io_size}" + done + done + for group_instance in $(seq 0 $(("${group_b_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}" "${io_count}" "${io_size}" + done + done + + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + done + + for group_instance in $(seq 0 $(("${group_a_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}" + done + done + for group_instance in $(seq 0 $(("${group_b_count}"-1))); do + for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}" + done + done + + # disable and remove all groups + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + mirror_group_disable "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + group_remove "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + mirror_group_disable "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + group_remove "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" + done + + # check all groups have been deleted and remove images + for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}" + images_remove "${primary_cluster}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}" + done + for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}" + images_remove "${primary_cluster}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}" + done +} + +# mirror a group then remove an image from that group and add to a different mirrored group. +declare -a test_image_move_group_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}") + +test_image_move_group_scenarios=1 + +test_image_move_group() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local image_prefix=$4 + + local image_count=5 + local group0=test-group0 + local group1=test-group1 + + group_create "${primary_cluster}" "${pool}/${group0}" + group_create "${primary_cluster}" "${pool}/${group1}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" + group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" "${image_count}" + + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" -testlog "TEST: empty group" -test_empty_group "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}" -testlog "TEST: empty group with namespace" -test_empty_group "${CLUSTER2}" "${CLUSTER1}" "${pool0}/${NS1}" "${group0}" + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0 + fi -testlog "TEST: create group with images then enable mirroring. Remove group without disabling mirroring" -test_create_group_with_images_then_mirror "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false' + local io_count=10240 + local io_size=4096 -testlog "TEST: create group with images then enable mirroring. Disable mirroring then remove group" -test_create_group_with_images_then_mirror "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' + # write to every image in the group, mirror group + for loop_instance in $(seq 0 $(("${image_count}"-1))); do + write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}" + done + mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group0}" + + # remove an image from the group and add to a different group + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}4" + else + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}4" + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + fi -testlog "TEST: create group then enable mirroring before adding images to the group. Remove group without disabling mirroring" -test_create_group_mirror_then_add_images "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false' + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}4" + mirror_group_enable "${primary_cluster}" "${pool}/${group1}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 1 + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 1 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 1 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-1)) + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-1)) + + # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled) + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}2" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}2" + else + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + mirror_group_disable "${primary_cluster}" "${pool}/${group1}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group1}" + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}2" + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}2" + mirror_group_enable "${primary_cluster}" "${pool}/${group1}" + fi -testlog "TEST: create group then enable mirroring before adding images to the group. Disable mirroring then remove group" -test_create_group_mirror_then_add_images "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' + wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 2 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 2 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 2 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-2)) + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-2)) + + echo "stopping daemon" + stop_mirrors "${secondary_cluster}" + + # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled) with the mirror daemon stopped + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}0" + else + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + mirror_group_disable "${primary_cluster}" "${pool}/${group1}" + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}0" + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + mirror_group_enable "${primary_cluster}" "${pool}/${group1}" + fi -testlog "TEST: two empty groups" -test_empty_groups "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${group1}" + echo "starting daemon" + start_mirrors "${secondary_cluster}" + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 3 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 3 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 3 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-3)) + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-3)) + + echo "stopping daemon" + stop_mirrors "${secondary_cluster}" + + # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled) with the mirror daemon stopped + # this time the moved image is still present in a snapshot for the old group that needs syncing and in a snapshot for the new group + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + + if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}1" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}1" + else + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + mirror_group_disable "${primary_cluster}" "${pool}/${group1}" + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}1" + group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}1" + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + mirror_group_enable "${primary_cluster}" "${pool}/${group1}" + fi -# testlog "TEST: add image from a different pool to group and test replay" -# different pools - not MVP -# test_images_different_pools "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${pool1}" "${group0}" "${image_prefix}" + echo "starting daemon" + start_mirrors "${secondary_cluster}" -testlog "TEST: create regular group snapshots and test replay" -test_create_group_with_images_then_mirror_with_regular_snapshots "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 4 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 4 + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 4 + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-4)) + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-4)) -testlog "TEST: add a large image to group and test replay" -test_create_group_with_large_image "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" + # set up a chain of moves TODO + + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + group_remove "${primary_cluster}" "${pool}/${group0}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + + mirror_group_disable "${primary_cluster}" "${pool}/${group1}" + group_remove "${primary_cluster}" "${pool}/${group1}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group1}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group1}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" +} -#TODO - test with mirrored images not in group +# test force promote scenarios +# TODO first two scenarios require support for dynamic groups +#declare -a test_force_promote_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_add') +#declare -a test_force_promote_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_remove') +declare -a test_force_promote_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_rename') +declare -a test_force_promote_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_expand') +declare -a test_force_promote_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_shrink') +declare -a test_force_promote_4=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'no_change') + +test_force_promote_scenarios=4 + +test_force_promote() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local image_prefix=$4 + local scenario=$5 + + local image_count=5 + local group0=test-group0 + local snap0='snap_0' + local snap1='snap_1' + + group_create "${primary_cluster}" "${pool}/${group0}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" $(("${image_count}"-1)) + write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" $(("${image_count}"-1)) + create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap0}" + compare_image_with_snapshot "${primary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + big_image=test-image-big + image_create "${primary_cluster}" "${pool}/${big_image}" 4G + group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${big_image}" + + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" + + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0 + fi + + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" + compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap1}" + + # make some changes to the big image so that the next sync will take a long time + write_image "${primary_cluster}" "${pool}" "${big_image}" 1024 4194304 + + local global_id + local image_size + local test_image_size + if [ "${scenario}" = 'image_add' ]; then + new_image=test-image-new + image_create "${primary_cluster}" "${pool}/${new_image}" + group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${new_image}" + get_image_mirroring_global_id "${primary_cluster}" "${pool}/${new_image}" global_id + test_image_with_global_id_present "${primary_cluster}" "${pool}" "${new_image}" "${global_id}" + test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${new_image}" "${global_id}" + elif [ "${scenario}" = 'image_remove' ]; then + get_image_mirroring_global_id "${primary_cluster}" "${pool}/${image_prefix}0" global_id + test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0" + test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + elif [ "${scenario}" = 'image_rename' ]; then + get_image_mirroring_global_id "${primary_cluster}" "${pool}/${image_prefix}0" global_id + image_rename "${primary_cluster}" "${pool}/${image_prefix}0" "${pool}/${image_prefix}_renamed_0" + test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}" + test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}" + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + elif [ "${scenario}" = 'image_expand' ]; then + get_image_size "${primary_cluster}" "${pool}/${image_prefix}2" image_size + image_resize "${primary_cluster}" "${pool}/${image_prefix}2" $((("${image_size}"/1024/1024)+4)) + test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}2" $(("${image_size}"+4*1024*1024)) + test_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}2" "${image_size}" + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + elif [ "${scenario}" = 'image_shrink' ]; then + get_image_size "${primary_cluster}" "${pool}/${image_prefix}3" image_size + image_resize "${primary_cluster}" "${pool}/${image_prefix}3" $((("${image_size}"/1024/1024)-4)) '--allow-shrink' + test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}3" $(("${image_size}"-4*1024*1024)) + test_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}3" "${image_size}" + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + elif [ "${scenario}" = 'no_change' ]; then + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + fi -: ' # TODO next test needs finishing -testlog "TEST: multiple images in group with io" -test_create_group_with_multiple_images_do_io "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" + # TODO add the following test +: ' + # This test removes and recreates an image - it fails currently as the request to list the group snaps on the secondary fails + group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0" + image_remove "${primary_cluster}" "${pool}/${image_prefix}0" + image_create "${primary_cluster}" "${pool}/${image_prefix}0" maybe different size? + group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0" ' + local group_snap_id + get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}/${group0}" group_snap_id + echo "id = ${group_snap_id}" + wait_for_test_group_snap_present "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1 + + if [ "${scenario}" = 'image_add' ]; then + wait_for_image_present "${secondary_cluster}" "${pool}" "${new_image}" 'present' + test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${new_image}" "${global_id}" + elif [ "${scenario}" = 'image_remove' ]; then + wait_for_image_present "${secondary_cluster}" "${pool}" "${image_prefix}0" 'deleted' + test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" $(("${image_count}"-1)) + elif [ "${scenario}" = 'image_rename' ]; then + wait_for_image_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" 'present' + test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}" + elif [ "${scenario}" = 'image_expand' ]; then + wait_for_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}2" $(("${image_size}"+4*1024*1024)) + elif [ "${scenario}" = 'image_shrink' ]; then + wait_for_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}3" $(("${image_size}"-4*1024*1024)) + fi + + # stop the daemon to prevent further syncing of snapshots + stop_mirrors "${secondary_cluster}" + + # check that latest snap is incomplete + ## this fails in the delete case as follows: + ##CEPH_ARGS='--id mirror' rbd --cluster cluster1 group snap list mirror/group_0 + ##ERR: rc= 2 + test_group_snap_sync_incomplete "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}" + + # force promote the group on the secondary - should rollback to the last complete snapshot + local old_primary_cluster + mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force' + old_primary_cluster="${primary_cluster}" + primary_cluster="${secondary_cluster}" + + mirror_group_demote "${old_primary_cluster}" "${pool}/${group0}" + secondary_cluster="${old_primary_cluster}" + + # Check that the rollback reverted the state + if [ "${scenario}" = 'image_add' ]; then + # check that new image is not present + test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${new_image}" "${global_id}" + elif [ "${scenario}" = 'image_remove' ]; then + test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + elif [ "${scenario}" = 'image_rename' ]; then + # check that the image is back with the original name + test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}" + test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}" + elif [ "${scenario}" = 'image_expand' ]; then + test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}2" "${image_size}" || fail "size mismatch" + elif [ "${scenario}" = 'image_shrink' ]; then + test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}3" "${image_size}" || fail "size mismatch" + fi + + + mirror_group_resync ${secondary_cluster} ${pool}/${group0} + + start_mirrors "${secondary_cluster}" + sleep 5 +# TODO check that data can be copied back to original primary cluster +# next line fails because latest snapshot on primary is never copied back to secondary +# finish off the resync function +# check that tidy up steps below work + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" + + compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + # Check that snapshots work on the new primary + mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" group_snap_id + wait_for_group_snap_present "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}" + wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}" + + # tidy up + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + group_remove "${primary_cluster}" "${pool}/${group0}" + + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" +} + +# test force promote scenarios +declare -a test_multiple_user_snapshot_time_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}") + +test_multiple_user_snapshot_time_scenarios=1 + +test_multiple_user_snapshot_time() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + + local image_count + local image_counts=(2 6) + local results=() + local time + + for image_count in "${image_counts[@]}"; do + test_multiple_user_snapshot_whilst_stopped "${primary_cluster}" "${secondary_cluster}" "${pool}" "${image_count}" time + results+=(${time}) + done + + for i in "${#results[@]}"; do + echo -e "${RED}image count:"$image_counts[$i]" snapshot time:"${results[$i]}"${NO_COLOUR}" + done + + if [ ${results[1]} -gt $((${results[0]}+2)) ]; then + fail "Snapshot time isn't independent of the group image count" + fi +} + +# test force promote scenarios +declare -a test_multiple_user_snapshot_whilst_stopped_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 5) + +test_multiple_user_snapshot_whilst_stopped_scenarios=1 + +test_multiple_user_snapshot_whilst_stopped() +{ + local primary_cluster=$1 ; shift + local secondary_cluster=$1 ; shift + local pool=$1 ; shift + local image_count=$1 ; shift + if [ -n "$1" ]; then + local get_average='true' + local -n _average_snapshot_time=$1 ; shift + fi + + local group0=test-group0 + local image_prefix="test_image" + + group_create "${primary_cluster}" "${pool}/${group0}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" 12M + group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" "${image_count}" + + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}" + + echo "stopping daemon on secondary" + stop_mirrors "${secondary_cluster}" + # TODO starting the daemon on the primary seem to cause a problem with image deletion - Nithya investigating (see slack thread) + #echo "starting daemon on primary" + #start_mirrors "${primary_cluster}" + + local start_time end_time + local times_result_arr=() + for i in {0..9}; do + start_time=$(date +%s) + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" + end_time=$(date +%s) + echo -e "mirror group snapshot time ="$((end_time-start_time)) + times_result_arr+=($((end_time-start_time))) + done; + + if [ -n "$get_average" ]; then + local cnt total + total=0 + cnt=0 + # ignore the times of the first 5 snapshots then determine the average of the rest + for i in {5..9}; do + total=$((total + times_result_arr[$i])) + cnt=$((cnt + 1)) + done; + echo "average=$((total/cnt))" + _average_snapshot_time=$((total/cnt)) + fi + + local count + get_group_snap_count "${primary_cluster}" "${pool}"/"${group0}" '*' count + test "${count}" -gt 3 || { fail "snap count = ${count}"; return 1; } + + get_group_snap_count "${secondary_cluster}" "${pool}"/"${group0}" '*' count + test "${count}" -eq 1 || { fail "snap count = ${count}"; return 1; } + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" + echo "starting daemon on secondary" + start_mirrors "${secondary_cluster}" + + # TODO remove this if when group successfully moves to synced state + if [ -z "$get_average" ]; then + # TODO this fails with the last image incomplete + wait_for_group_synced "${primary_cluster}" "${pool}/${group0}" + + get_group_snap_count "${primary_cluster}" "${pool}"/"${group0}" '*' count + get_group_snap_count "${secondary_cluster}" "${pool}"/"${group0}" '*' count + + # TODO this fails with the image on the secondary in split-brain + wait_for_status_in_pool_dir "${secondary_cluster}" "${pool}" "${image_prefix}" 'up+replaying' + fi + + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" + wait_for_group_present "${primary_cluster}" "${pool}" "${group0}" "${image_count}" + #mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + group_remove "${primary_cluster}" "${pool}/${group0}" + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" +} + +# test resync scenarios +declare -a test_resync_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'no_change') + +test_resync_scenarios=1 + +test_resync() +{ + local primary_cluster=$1 + local secondary_cluster=$2 + local pool=$3 + local image_prefix=$4 + local scenario=$5 + + local image_count=5 + local group0=test-group0 + local snap0='snap_0' + local snap1='snap_1' + + group_create "${primary_cluster}" "${pool}/${group0}" + images_create "${primary_cluster}" "${pool}/${image_prefix}" $(("${image_count}")) + write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" $(("${image_count}")) + + create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap0}" + compare_image_with_snapshot "${primary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + mirror_group_enable "${primary_cluster}" "${pool}/${group0}" + wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}" + wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}" + wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}" + + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0 + fi + + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" + compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + local group_snap_id secondary_group_snap_id primary_group_snap_id + get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}/${group0}" primary_group_snap_id + echo "id = ${primary_group_snap_id}" + + # stop the daemon to prevent further syncing of snapshots + stop_mirrors "${secondary_cluster}" + + # promote secondary and change data on image + mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force' + write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + # demote secondary again + mirror_group_demote "${secondary_cluster}" "${pool}/${group0}" + + # restart daemon and request a resync from primary + start_mirrors "${secondary_cluster}" + mirror_group_resync ${secondary_cluster} ${pool}/${group0} + + # confirm that data on secondary again matches initial snapshot on primary + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" + compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + # Repeat the test this time changing the data on the primary too. + + # stop the daemon to prevent further syncing of snapshots + stop_mirrors "${secondary_cluster}" + + # promote secondary and change data on image + mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force' + write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" group_snap_id + + # demote secondary again + mirror_group_demote "${secondary_cluster}" "${pool}/${group0}" + + # restart daemon and request a resync from primary + start_mirrors "${secondary_cluster}" + mirror_group_resync ${secondary_cluster} ${pool}/${group0} + + # confirm that data on secondary again matches latest snapshot on primary + wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" + wait_for_test_group_snap_present "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1 + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}0" + + # Repeat the test this time swapping the primary and secondary and resyncing back to the new secondary. +: ' + # TODO this test fails - does not sync back to old primary + + # stop the daemon to prevent further syncing of snapshots + stop_mirrors "${secondary_cluster}" + + # promote secondary and change data on image + mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force' + write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + + compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}" + + # change data on old primary too + write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096 + mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" group_snap_id + + # demote old primary + mirror_group_demote "${primary_cluster}" "${pool}/${group0}" + + # restart daemon and request a resync from primary + start_mirrors "${primary_cluster}" + mirror_group_resync ${primary_cluster} ${pool}/${group0} + + # confirm that data on secondary again matches latest snapshot on primary + wait_for_group_synced "${secondary_cluster}" "${pool}"/"${group0}" + wait_for_test_group_snap_present "${primary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1 + compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}0" + +' + + mirror_group_disable "${primary_cluster}" "${pool}/${group0}" + group_remove "${primary_cluster}" "${pool}/${group0}" + + wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}" + wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}" + + images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" +} + +run_test() +{ + local test_name=$1 + local test_scenario=$2 + + declare -n test_parameters="$test_name"_"$test_scenario" + + testlog "TEST:$test_name scenario:$test_scenario parameters:" "${test_parameters[@]}" + "$test_name" "${test_parameters[@]}" +} + +# exercise all scenarios that are defined for the specified test +run_test_scenarios() +{ + local test_name=$1 + + declare -n test_scenario_count="$test_name"_scenarios + + local loop + for loop in $(seq 1 $test_scenario_count); do + run_test $test_name $loop + done +} + +# exercise all scenarios for all tests +run_tests() +{ + run_test_scenarios test_empty_group + run_test_scenarios test_empty_groups + # This next test requires support for dynamic groups TODO + # run_test_scenarios test_mirrored_group_remove_all_images + # This next test is unreliable - image/group ends up in stopped also requires dynamic groups - TODO enable + # run_test_scenarios test_mirrored_group_add_and_remove_images + # This next test is unreliable - image ends up in stopped also requires dynamic groups - TODO enable + # run_test_scenarios test_create_group_mirror_then_add_images + run_test_scenarios test_create_group_with_images_then_mirror + # next test is not MVP - TODO + # run_test_scenarios test_images_different_pools + # TODO next test fails if run with other tests - seems to have passed on its own must retry + # run_test_scenarios test_create_group_with_images_then_mirror_with_regular_snapshots + run_test_scenarios test_create_group_with_large_image + #run_test_scenarios test_create_group_with_multiple_images_do_io + run_test_scenarios test_group_and_standalone_images_do_io + run_test_scenarios test_create_multiple_groups_do_io + #run_test_scenarios test_stopped_daemon + #run_test_scenarios test_create_group_with_regular_snapshots_then_mirror + #run_test_scenarios test_image_move_group + #run_test_scenarios test_force_promote + #run_test_scenarios test_resync + run_test_scenarios test_remote_namespace + #run_test_scenarios test_multiple_user_snapshot_whilst_stopped + #run_test_scenarios test_create_group_with_image_remove_then_repeat + #run_test_scenarios test_enable_disable_repeat + #run_test_scenarios test_empty_group_omap_keys + #run_test_scenarios test_group_with_clone_image + #run_test_scenarios test_multiple_user_snapshot_time +} + +if [ -n "${RBD_MIRROR_SHOW_CMD}" ]; then + set -e +else + set -ex +fi + +# If the tmpdir and cluster conf file exist then reuse the existing cluster +if [ -d "${RBD_MIRROR_TEMDIR}" ] && [ -f "${RBD_MIRROR_TEMDIR}"'/cluster1.conf' ] +then + export RBD_MIRROR_USE_EXISTING_CLUSTER=1 +fi + +setup + +# see if we need to (re)start rbd-mirror deamon +pid=$(cat "$(daemon_pid_file "${CLUSTER1}")" 2>/dev/null) || : +if [ -z "${pid}" ] +then + start_mirrors "${CLUSTER1}" +fi +check_daemon_running "${CLUSTER1}" + +# restore the arguments from the cli +set -- "${args[@]}" + +# loop count is specified as first argument. default value is 1 +loop_count="${1:-1}" +for loop in $(seq 1 "${loop_count}"); do + echo "run number ${loop} of ${loop_count}" + if [ "$#" -gt 2 ] + then + # second arg is test_name + # third arg is scenario number + # fourth arg defines RBD_IMAGE_FEATURES (see top of file) + run_test "$2" "$3" + else + run_tests + fi +done + exit 0 diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index f4d0eb818e8..f5775d61211 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -107,6 +107,7 @@ if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then fi RED='\033[0;31m' +GREEN='\033[0;32m' NO_COLOUR='\033[0m' export CEPH_ARGS="--id ${CEPH_ID}" @@ -254,8 +255,9 @@ fail() { fi if [ -n "${fatal}" ]; then - echo "${fatal}" 1>&2 + echo -e "${RED}${fatal}" 1>&2 print_stacktrace + echo -e "${NO_COLOUR}" exit 1 fi @@ -408,6 +410,17 @@ setup_pools() local admin_key_file local uuid + # Create and delete a random number of pools, images and snapshots so that ids on the two clusters sometimes mismatch + pool_count=$(( RANDOM % 2 )) + for loop_instance in $(seq 0 ${pool_count}); do + run_admin_cmd "ceph --cluster ${cluster} osd pool create dummy_pool 64 64" + image_create "${cluster}" "dummy_pool/dummy_image" + create_snapshot "${cluster}" "dummy_pool" "dummy_image" "dummy_snap" + group_create "${cluster}" "dummy_pool/dummy_group" + group_snap_create "${cluster}" "dummy_pool/dummy_group" "dummy_snap" + run_admin_cmd "ceph --cluster ${cluster} osd pool delete dummy_pool dummy_pool --yes-i-really-really-mean-it" + done + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64 CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64 @@ -492,8 +505,7 @@ cleanup() local error_code=$1 set +e - - if [ "${error_code}" -ne 0 ]; then + if [ "${error_code}" -ne 0 ] && [ -z "${RBD_MIRROR_NO_STATUS}" ]; then status fi @@ -534,7 +546,7 @@ start_mirror() set_cluster_instance "${cluster}" cluster instance test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return - local log=${TEMPDIR}/rbd-mirror${instance}.out + local log=${TEMPDIR}/rbd-mirror-${cluster}-${instance}.out ulimit -c unlimited rbd-mirror \ @@ -959,7 +971,6 @@ wait_for_snapshot_sync_complete() local status_log=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.status) local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.status) - mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}" get_newest_mirror_snapshot "${cluster}" "${remote_pool}" "${image}" "${status_log}" local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) @@ -989,12 +1000,104 @@ wait_for_replay_complete() if [ "${RBD_MIRROR_MODE}" = "journal" ]; then wait_for_journal_replay_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image} elif [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}" wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image} else return 1 fi } +count_fields_in_mirror_pool_status() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local -n _pool_result_count_arr=$1 ; shift + local fields=("$@") + + run_cmd "rbd --cluster ${cluster} mirror pool status --verbose ${pool} --format xml --pretty-format" || { fail; return 1; } + + local field result + for field in "${fields[@]}"; do + result=$($XMLSTARLET sel -t -v "count($field)" < "$CMD_STDOUT") + _pool_result_count_arr+=( "${result}" ) + done +} + +get_fields_from_mirror_pool_status() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local -n _pool_result_arr=$1 ; shift + local fields=("$@") + + run_cmd "rbd --cluster ${cluster} mirror pool status --verbose ${pool} --format xml --pretty-format" || { fail; return 1; } + + local field result + for field in "${fields[@]}"; do + result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; } + _pool_result_arr+=( "${result}" ) + done +} + +get_fields_from_mirror_group_status() +{ + local cluster=$1 ; shift + local group_spec=$1 ; shift + local -n _group_result_arr=$1 ; shift + local fields=("$@") + + run_admin_cmd "rbd --cluster ${cluster} mirror group status ${group_spec} --format xml --pretty-format" || { fail; return 1; } + + local field result + for field in "${fields[@]}"; do + result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; } + _group_result_arr+=( "${result}" ) + done +} + +get_fields_from_mirror_image_status() +{ + local cluster=$1 ; shift + local image_spec=$1 ; shift + local -n _image_result_arr=$1 ; shift + local fields=("$@") + + run_admin_cmd "rbd --cluster ${cluster} mirror image status ${image_spec} --format xml --pretty-format" || { fail; return 1; } + + local field result + for field in "${fields[@]}"; do + result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; } + _image_result_arr+=( "${result}" ) + done +} + +check_fields_in_group_and_image_status() +{ + local cluster=$1 + local group_spec=$2 + + local fields=(//group/state //group/description) + local group_fields_arr + get_fields_from_mirror_group_status "${cluster}" "${group_spec}" group_fields_arr "${fields[@]}" + + local image_spec + for image_spec in $(rbd --cluster "${cluster}" group image list "${group_spec}" | xargs); do + local fields=(//image/state //image/description) + local image_fields_arr + get_fields_from_mirror_image_status "${cluster}" "${image_spec}" image_fields_arr "${fields[@]}" + + # check that the image "state" matches the group "state" +# TODO. The imaage status doesn not always get updated before the group status - see slack thread. Fail and allow retry for now +# test "${image_fields_arr[0]}" = "${group_fields_arr[0]}" || { fail "image:${image_spec} ${image_fields_arr[0]} != ${group_fields_arr[0]}"; return 1; } + test "${image_fields_arr[0]}" = "${group_fields_arr[0]}" || { fail; return 1; } + + # check that the image "description" matches the group "description". Need to remove the extra information from the image description first + local image_description + image_description=$(cut -d ',' -f 1 <<< "${image_fields_arr[1]}") +# test "${image_description}" = "${group_fields_arr[1]}" || { fail "image:${image_spec} ${image_description} != ${group_fields_arr[1]}"; return 1; } + test "${image_description}" = "${group_fields_arr[1]}" || { fail; return 1; } + done +} test_status_in_pool_dir() { @@ -1206,6 +1309,15 @@ rename_image() rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name} } +image_rename() +{ + local cluster=$1 + local src_image_spec=$2 + local dst_image_spec=$3 + + run_cmd "rbd --cluster=${cluster} rename ${src_image_spec} ${dst_image_spec}" +} + remove_image() { local cluster=$1 @@ -1567,6 +1679,43 @@ compare_image_snapshots() return ${ret} } +compare_image_with_snapshot() +{ + local img_cluster=$1 ; shift + local image_spec=$1 ; shift + local snap_cluster=$1 ; shift + local snap_spec=$1 ; shift + + if [ -n "$1" ]; then + expect_difference=$1 ; shift + fi + + local ret=0 + + local img_export snap_export + img_export=${TEMPDIR}/$(mkfname ${img_cluster}-${image_spec}.export) + snap_export=${TEMPDIR}/$(mkfname ${snap_cluster}-${snap_spec}.export) + rm -f "${img_export}" "${snap_export}" + + rbd --cluster "${img_cluster}" export "${image_spec}" "${img_export}" + rbd --cluster "${snap_cluster}" export "${snap_spec}" "${snap_export}" + + if ! cmp "${img_export}" "${snap_export}" + then + if [ 'true' != "${expect_difference}" ]; then + show_diff "${img_export}" "${snap_export}" + ret=1 + fi + fi + rm -f "${img_export}" "${snap_export}" + return "${ret}" +} + +compare_image_with_snapshot_expect_difference() +{ + compare_image_with_snapshot "$@" 'true' +} + demote_image() { local cluster=$1 @@ -1619,9 +1768,9 @@ enable_mirror() local image=$3 local mode=${4:-${RBD_MIRROR_MODE}} - rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} + run_cmd "rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode}" # Display image info including the global image id for debugging purpose - rbd --cluster=${cluster} info ${pool}/${image} + run_cmd "rbd --cluster=${cluster} info ${pool}/${image}" } test_image_present() @@ -1642,6 +1791,65 @@ test_image_present() test "${test_state}" = "${current_state}" } +test_image_with_global_id_count() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local global_id=$4 + local test_image_count=$5 + + run_cmd "rbd --cluster ${cluster} info ${pool}/${image} --format xml --pretty-format" + test "${test_image_count}" = "$($XMLSTARLET sel -t -v "count(//image/mirroring[global_id='${global_id}'])" < "$CMD_STDOUT")" || { fail; return 1; } +} + +test_image_count() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local test_image_count=$4 + + run_cmd "rbd --cluster ${cluster} ls ${pool} --format xml --pretty-format" + test "${test_image_count}" = "$($XMLSTARLET sel -t -v "count(//images[name='${image}'])" < "$CMD_STDOUT")" || { fail; return 1; } +} + +test_image_with_global_id_not_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local global_id=$4 + + # if the image is not listed in the pool then no need to check the global id + test_image_count "${cluster}" "${pool}" "${image}" 0 && return 0; + + test_image_with_global_id_count "${cluster}" "${pool}" "${image}" "${global_id}" 0 || { fail "image present"; return 1; } +} + +test_image_with_global_id_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local global_id=$4 + + # if the image is not listed in the pool then no need to check the global id + test_image_count "${cluster}" "${pool}" "${image}" 1 || return 1; + + test_image_with_global_id_count "${cluster}" "${pool}" "${image}" "${global_id}" 1 +} + +test_image_not_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local image_id=$4 + + test_image_present "${cluster}" "${pool}" "${image}" 'deleted' "${image_id}" +} + wait_for_image_present() { local cluster=$1 @@ -1674,6 +1882,60 @@ get_image_id() sed -ne 's/^.*block_name_prefix: rbd_data\.//p' } +get_image_mirroring_global_id() +{ + local cluster=$1 + local image_spec=$2 + local -n _global_id=$3 + + run_cmd "rbd --cluster ${cluster} info ${image_spec} --format xml --pretty-format" + _global_id=$($XMLSTARLET sel -t -v "//image/mirroring/global_id" "$CMD_STDOUT") || { fail "not mirrored"; return; } +} + +image_resize() +{ + local cluster=$1 ; shift + local image_spec=$1 ; shift + local size=$1 ; shift + + run_cmd "rbd --cluster ${cluster} resize --image ${image_spec} --size ${size} $*" +} + +get_image_size() +{ + local cluster=$1 + local image_spec=$2 + local -n _size=$3 + + run_cmd "rbd --cluster ${cluster} info ${image_spec} --format xml --pretty-format" + _size=$($XMLSTARLET sel -t -v "//image/size" "$CMD_STDOUT") || { fail "unable to determine size"; return; } +} + +test_image_size_matches() +{ + local cluster=$1 + local image_spec=$2 + local test_size=$3 + + local current_size + get_image_size "${cluster}" "${image_spec}" current_size + test "${current_size}" = "${test_size}" || { fail; return 1; } +} + +wait_for_image_size_matches() +{ + local cluster=$1 + local image_spec=$2 + local test_size=$3 + local s + + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + test_image_size_matches "${cluster}" "${image_spec}" "${test_size}" && return 0 + done + fail "size never matched"; return 1 +} + request_resync_image() { local cluster=$1 @@ -1724,8 +1986,7 @@ list_omap_keys() local cluster=$1 local pool=$2 local obj_name=$3 - - rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name} + run_cmd "rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name}" } count_omap_keys_with_filter() @@ -1734,8 +1995,10 @@ count_omap_keys_with_filter() local pool=$2 local obj_name=$3 local filter=$4 + local -n _count=$5 - list_omap_keys ${cluster} ${pool} ${obj_name} | grep -c ${filter} + list_omap_keys "${cluster}" "${pool}" "${obj_name}" + _count=$(grep -c "${filter}" "$CMD_STDOUT") || return 0 } wait_for_omap_keys() @@ -1745,19 +2008,13 @@ wait_for_omap_keys() local obj_name=$3 local filter=$4 + local key_count for s in 0 1 2 2 4 4 8 8 8 16 16 32; do sleep $s - - set +e - test "$(count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter})" = 0 - error_code=$? - set -e - - if [ $error_code -eq 0 ]; then - return 0 - fi + count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter} key_count + test "${key_count}" = 0 && return 0 done - + fail "wait for count of keys 0 failed on ${cluster}. Actual count=${key_count}" return 1 } @@ -1796,6 +2053,37 @@ group_remove() run_cmd "rbd --cluster ${cluster} group remove ${group_spec}" } +wait_for_group_synced() +{ + local cluster=$1 + local group_spec=$2 + + # Determine secondary cluster + local secondary_cluster + get_fields_from_mirror_group_status "${cluster}" "${group_spec}" secondary_cluster "(//group/peer_sites/peer_site/site_name)" + + local secondary_group_spec + IFS='/' read -r -a group_fields <<< "${group_spec}" + if [ "${#group_fields[@]}" -eq 3 ]; then + local pool local_namespace secondary_namespace group + pool="${group_fields[0]}" + local_namespace="${group_fields[1]}" + group="${group_fields[2]}" + + # Determine secondary cluster namespace + run_cmd "rbd --cluster ${cluster} mirror pool info ${pool}/${local_namespace} --format xml --pretty-format" || { fail; return 1; } + secondary_namespace=$(xmlstarlet sel -t -v "//${pool}/remote_namespace" < ${CMD_STDOUT}) || { fail "no remote namespace"; return 1; } + secondary_group_spec="${pool}/${secondary_namespace}/${group}" + else + secondary_group_spec="${group_spec}" + fi + + local group_snap_id + get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id + wait_for_group_snap_present "${secondary_cluster}" "${secondary_group_spec}" "${group_snap_id}" + wait_for_group_snap_sync_complete "${secondary_cluster}" "${secondary_group_spec}" "${group_snap_id}" +} + group_image_add() { local cluster=$1 @@ -1840,13 +2128,29 @@ group_images_remove() done } +mirror_group_internal() +{ + local cmd=$1 + local group_spec=$2 + local mode=${3:-${MIRROR_IMAGE_MODE}} + + run_cmd "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}" +} + mirror_group_enable() { local cluster=$1 local group_spec=$2 local mode=${3:-${MIRROR_IMAGE_MODE}} + local runner=${4:-"run_cmd"} - run_cmd "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}" + "$runner" "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}" +} + +mirror_group_enable_try() +{ + local mode=${3:-${MIRROR_IMAGE_MODE}} + mirror_group_enable "$@" "${mode}" "try_cmd" } mirror_group_disable() @@ -1889,7 +2193,13 @@ mirror_group_snapshot() local cluster=$1 local group_spec=$2 - run_cmd "rbd --cluster=${cluster} mirror group snapshot ${group_spec}" + run_cmd "rbd --cluster=${cluster} mirror group snapshot ${group_spec}" || return 1 + + if [ "$#" -gt 2 ] + then + local -n _group_snap_id=$3 + _group_snap_id=$(awk -F': ' '{print $NF}' "$CMD_STDOUT" ) + fi } group_snap_create() @@ -1916,9 +2226,58 @@ get_group_snap_count() local group_spec=$2 local snap=$3 local -n _group_snap_count=$4 + + run_cmd "rbd --cluster=${cluster} group snap ls --format xml --pretty-format ${group_spec}" + if [ "${snap}" = '*' ]; then + _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap)" < "$CMD_STDOUT")" + else + _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[snapshot='${snap}'])" < "$CMD_STDOUT")" + fi +} + +get_group_snap_name() +{ + local cluster=$1 + local group_spec=$2 + local snap_id=$3 + local -n _group_snap_name=$4 run_cmd "rbd --cluster=${cluster} group snap ls --format xml --pretty-format ${group_spec}" - _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[snapshot='${snap}'])" < "$CMD_STDOUT")" + _group_snap_name="$($XMLSTARLET sel -t -v "//group_snaps/group_snap[id='${snap_id}']/snapshot" < "$CMD_STDOUT")" +} + +get_image_snap_id_from_group_snap_info() +{ + local cluster=$1 + local snap_spec=$2 + local image_spec=$3 + local -n _image_snap_id=$4 + + run_cmd "rbd --cluster=${cluster} group snap info --format xml --pretty-format ${snap_spec}" + local image_name + image_name=$(echo "${image_spec}" | awk -F'/' '{print $NF}') + _image_snap_id="$($XMLSTARLET sel -t -v "//group_snapshot/images/image[image_name='${image_name}']/snap_id" < "$CMD_STDOUT")" +} + +get_images_from_group_snap_info() +{ + local cluster=$1 + local snap_spec=$2 + local -n _images=$3 + + run_cmd "rbd --cluster=${cluster} group snap info --format xml --pretty-format ${snap_spec}" + # sed script removes extra path delimiter if namespace field is blank + _images="$($XMLSTARLET sel -t -m "//group_snapshot/images/image" -v "pool_name" -o "/" -v "namespace" -o "/" -v "image_name" -o " " < "$CMD_STDOUT" | sed s/"\/\/"/"\/"/g )" +} + +get_image_snap_complete() +{ + local cluster=$1 + local image_spec=$2 + local snap_id=$3 + local -n _is_complete=$4 + run_cmd "rbd --cluster=${cluster} snap list --all --format xml --pretty-format ${image_spec}" + _is_complete="$($XMLSTARLET sel -t -v "//snapshots/snapshot[id='${snap_id}']/namespace/complete" < "$CMD_STDOUT")" } check_group_snap_doesnt_exist() @@ -2009,6 +2368,112 @@ wait_for_group_not_present() wait_for_test_group_present "${cluster}" "${pool}" "${group}" 0 0 } +test_group_snap_present() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + local expected_snap_count=$4 + + # TODO - have seen this next cmd fail with rc=2 and an empty list + # this should not happen, but if it does then retry as a temp workaround + try_cmd "rbd --cluster ${cluster} group snap list ${group_spec} --format xml --pretty-format" + + test "${expected_snap_count}" = "$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[id='${group_snap_id}'])" < "$CMD_STDOUT")" || { fail; return 1; } +} + +wait_for_test_group_snap_present() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + local test_group_snap_count=$4 + local s + + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" "${test_group_snap_count}" && return 0 + done + + fail "wait for count of group snaps with id ${group_snap_id} to be ${test_group_snap_count} failed on ${cluster}" + return 1 +} + +wait_for_group_snap_present() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + + wait_for_test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" 1 +} + +wait_for_group_snap_not_present() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + + wait_for_test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" 0 +} + +test_group_snap_sync_state() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + local expected_state=$4 + + # TODO - have seen this next cmd fail with rc=2 and an empty list + # this should not happen, but if it does then retry as a temp workaround + try_cmd "rbd --cluster ${cluster} group snap list ${group_spec} --format xml --pretty-format" + + test "${expected_state}" = "$($XMLSTARLET sel -t -v "//group_snaps/group_snap[id='${group_snap_id}']/state" < "$CMD_STDOUT")" || { fail; return 1; } +} + +test_group_snap_sync_complete() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + + test_group_snap_sync_state "${cluster}" "${group_spec}" "${group_snap_id}" 'complete' +} + +test_group_snap_sync_incomplete() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + + test_group_snap_sync_state "${cluster}" "${group_spec}" "${group_snap_id}" 'incomplete' +} + +wait_for_test_group_snap_sync_complete() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + local s + + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + test_group_snap_sync_complete "${cluster}" "${group_spec}" "${group_snap_id}" && return 0 + done + + fail "wait for group snap with id ${group_snap_id} to be synced failed on ${cluster}" + return 1 +} + +wait_for_group_snap_sync_complete() +{ + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + + wait_for_test_group_snap_sync_complete "${cluster}" "${group_spec}" "${group_snap_id}" +} + test_group_replay_state() { local cluster=$1 @@ -2036,6 +2501,25 @@ test_group_replay_state() fi } +query_replayer_assignment() +{ + local cluster=$1 + local instance=$2 + local -n _result=$3 + + local group_replayers + local image_replayers + local group_replayers_count + local image_replayers_count + + admin_daemon "${cluster}:${instance}" rbd mirror status --format xml-pretty || { fail; return 1; } + group_replayers=$($XMLSTARLET sel -t -v "//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/name" < "$CMD_STDOUT") || { group_replayers=''; } + image_replayers=$($XMLSTARLET sel -t -v "//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/image_replayers/image_replayer/name" < "$CMD_STDOUT") || { image_replayers=''; } + group_replayers_count=$($XMLSTARLET sel -t -v "count(//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/name)" < "$CMD_STDOUT") || { group_replayers_count='0'; } + image_replayers_count=$($XMLSTARLET sel -t -v "count(//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/image_replayers/image_replayer/name)" < "$CMD_STDOUT") || { image_replayers_count='0'; } + _result=("${group_replayers}" "${image_replayers}" "${group_replayers_count}" "${image_replayers_count}") +} + wait_for_group_replay_state() { local cluster=$1 @@ -2095,68 +2579,64 @@ get_newest_group_mirror_snapshot_id() mirror_group_snapshot_and_wait_for_sync_complete() { - local local_cluster=$1 - local cluster=$2 + local secondary_cluster=$1 + local primary_cluster=$2 local group_spec=$3 local group_snap_id - local local_group_snap_id if [ "${MIRROR_IMAGE_MODE}" != "snapshot" ]; then return 1 fi - mirror_group_snapshot "${cluster}" "${group_spec}" - get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id - - while true; do - for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do - sleep ${s} - - get_newest_group_mirror_snapshot_id "${local_cluster}" "${group_spec}" local_group_snap_id - test "${local_group_snap_id}" = "${group_snap_id}" && return 0 - done - fail "Failed to reach expected state" - return 1 - done - return 1 + mirror_group_snapshot "${primary_cluster}" "${group_spec}" group_snap_id + wait_for_group_snap_present "${secondary_cluster}" "${group_spec}" "${group_snap_id}" + wait_for_group_snap_sync_complete "${secondary_cluster}" "${group_spec}" "${group_snap_id}" } -test_group_and_image_sync_status() +test_group_synced_image_status() { - local local_cluster=$1 - local cluster=$2 - local group_spec=$3 - local image_spec=$4 + local cluster=$1 + local group_spec=$2 + local group_snap_id=$3 + local expected_synced_image_count=$4 - local group_snap_id - local local_group_snap_id + local group_snap_name + get_group_snap_name "${cluster}" "${group_spec}" "${group_snap_id}" group_snap_name - get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id + local images + get_images_from_group_snap_info "${cluster}" "${group_spec}@${group_snap_name}" images - local local_image_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${group_spec}-${image_spec}-${local_group_snap_id}.status) + local image_count=0 + local image_spec + for image_spec in ${images}; do - while true; do - for s in 0.4 1 1 1 2 2 2 4 8 16 16 32 32; do - sleep ${s} + # get the snap_id for this image from the group snap info + local image_snap_id + get_image_snap_id_from_group_snap_info "${cluster}" "${group_spec}@${group_snap_name}" "${image_spec}" image_snap_id + + # get the value in the "complete" field for the image and snap_id + local is_complete + get_image_snap_complete "${cluster}" "${image_spec}" "${image_snap_id}" is_complete - get_newest_group_mirror_snapshot_id "${local_cluster}" "${group_spec}" local_group_snap_id - if [ "${local_group_snap_id}" == "${group_snap_id}" ]; then -#TODO: Get the image snap for this group snap. For now, compare against the last mirror image .group snap. -# use rbd group snap info for this once it is working properly - rbd --cluster "${local_cluster}" snap list --all "${image_spec}" --format xml | \ - $XMLSTARLET sel -t -c "//snapshots/snapshot[namespace/type='mirror' and position()=last()]" > ${local_image_status_log} - local image_snap_complete=$(xmlstarlet sel -t -v "//snapshot/namespace/complete" < ${local_image_status_log}) - if [ "${image_snap_complete}" != "true" ]; then - return 1 - else - return 0 - fi - else - continue - fi - done - return 1 + test "${is_complete}" != "true" && { fail "image ${image_spec} is not synced"; return 1; } + + image_count=$((image_count+1)) done + + test "${image_count}" != "${expected_synced_image_count}" && fail "unexpected count ${image_count} != ${expected_synced_image_count}" + + return 0 +} + +test_images_in_latest_synced_group() +{ + local cluster=$1 + local group_spec=$2 + local expected_synced_image_count=$3 + + local group_snap_id + get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id + test_group_synced_image_status "${cluster}" "${group_spec}" "${group_snap_id}" "${expected_synced_image_count}" } test_group_status_in_pool_dir() @@ -2190,10 +2670,14 @@ test_group_status_in_pool_dir() test "${image_count}" = "${actual_image_count}" || { fail; return 1; } # If the group is started then check that all images are started too - test "${current_state}" = "started" || return 0 - test "${image_count}" = "${started_image_count}" || { fail; return 1; } + if [ "${current_state}" = "started" ]; then + test "${image_count}" = "${started_image_count}" || { fail; return 1; } + fi fi + # TODO enable this once tests are more reliable + #check_fields_in_group_and_image_status "${cluster}" "${group_spec}" || { fail; return 1; } + return 0 } @@ -2227,6 +2711,16 @@ tidy() stop_mirrors ${cluster} '-9' done + for cluster in ${primary_cluster} ${secondary_cluster}; do + echo 'cluster:'${cluster} + for pool in $(CEPH_ARGS='' ceph --cluster ${cluster} osd pool ls | grep -v "^\." | xargs); do + echo 'pool:'${pool} + run_admin_cmd "ceph --cluster ${cluster} osd pool delete ${pool} ${pool} --yes-i-really-really-mean-it" + done + done + + # following is old method that used to remove individual object rather than removing entire pools + : ' for cluster in ${primary_cluster} ${secondary_cluster}; do echo 'cluster:'${cluster} for pool in "${POOL}" "${PARENT_POOL}" "${POOL}/${NS1}" "${POOL}/${NS2}"; do @@ -2250,7 +2744,7 @@ tidy() done done done - + ' } # list all groups, images and snaps -- 2.39.5