-#!/usr/bin/env bash
+ #!/usr/bin/env bash
#
# rbd_mirror_group_simple.sh
#
# It may repeat some of the tests from rbd_mirror_group.sh, but only those that are known to work
# It has a number of extra tests that imclude multiple images in a group
#
+# shellcheck disable=SC2034 # Don't warn about unused variables and functions
+# shellcheck disable=SC2317 # Don't warn about unreachable commands
+
export RBD_MIRROR_NOCLEANUP=1
export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
export RBD_MIRROR_SHOW_CMD=1
export RBD_MIRROR_MODE=snapshot
+group0=test-group0
+group1=test-group1
+pool0=mirror
+pool1=mirror_parent
+image_prefix=test-image
+
+feature=${4:-0}
+features=(
+ "layering,exclusive-lock,object-map,fast-diff,deep-flatten"
+ "layering,exclusive-lock,object-map,fast-diff"
+ "layering,exclusive-lock"
+ "layering"
+)
+
+# save and clear the cli args (can't call rbd_mirror_helpers with these defined)
+args=("$@")
+set --
+
+
+if [ -z "${RBD_IMAGE_FEATURES}" ]; then
+ echo "RBD_IMAGE_FEATURES=${features[${feature}]}"
+ RBD_IMAGE_FEATURES=${features[${feature}]}
+fi
+
. $(dirname $0)/rbd_mirror_helpers.sh
+# create group with images then enable mirroring. Remove group without disabling mirroring
+declare -a test_create_group_with_images_then_mirror_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false' 5)
+# create group with images then enable mirroring. Disable mirroring then remove group
+declare -a test_create_group_with_images_then_mirror_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' 5)
+declare -a test_create_group_with_images_then_mirror_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true' 30)
+
+test_create_group_with_images_then_mirror_scenarios=3
+
test_create_group_with_images_then_mirror()
{
- local primary_cluster=$1
- local secondary_cluster=$2
- local pool=$3
- local group=$4
- local image_prefix=$5
- local disable_before_remove=$6
+ local primary_cluster=$1 ; shift
+ local secondary_cluster=$1 ; shift
+ local pool=$1 ; shift
+ local group=$1 ; shift
+ local image_prefix=$1 ; shift
+ local disable_before_remove=$1 ; shift
+ local image_count=$1 ; shift
+ if [ -n "$1" ]; then
+ local get_times='true'
+ local -n _times_result_arr=$8
+ fi
group_create "${primary_cluster}" "${pool}/${group}"
- images_create "${primary_cluster}" "${pool}/${image_prefix}" 5
- group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" 5
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${image_count}"
+ # write to every image in the group
+ local io_count=10240
+ local io_size=4096
+ for loop_instance in $(seq 0 $(("${image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+
+ local start_time enabled_time synced_time
+ start_time=$(date +%s)
mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ enabled_time=$(date +%s)
# rbd group list poolName (check groupname appears in output list)
# do this before checking for replay_started because queries directed at the daemon fail with an unhelpful
# error message before the group appears on the remote cluster
- wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 5
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${image_count}"
check_daemon_running "${secondary_cluster}"
# ceph --daemon mirror group status groupName
- wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 5
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${image_count}"
check_daemon_running "${secondary_cluster}"
# rbd mirror group status groupName
- wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 5
+ #sleep 10
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${image_count}"
+ wait_for_group_synced "${primary_cluster}" "${pool}/${group}"
+ synced_time=$(date +%s)
+ if [ -n "$get_times" ]; then
+ _times_result_arr+=($((enabled_time-start_time)))
+ _times_result_arr+=($((synced_time-enabled_time)))
+ fi
check_daemon_running "${secondary_cluster}"
if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
check_daemon_running "${secondary_cluster}"
if [ 'false' != "${disable_before_remove}" ]; then
- mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
fi
group_remove "${primary_cluster}" "${pool}/${group}"
wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
check_daemon_running "${secondary_cluster}"
- images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
+
+# record the time taken to enable and sync for a group with increasing number of images.
+declare -a test_group_enable_times_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}")
+
+test_group_enable_times_scenarios=1
+
+test_group_enable_times()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local disable_before_remove='true'
+ local image_count
+ local results=()
+ local times=()
+
+ for image_count in {0,5,10,15,20,25,30}; do
+ times=()
+ test_create_group_with_images_then_mirror "${primary_cluster}" "${secondary_cluster}" "${pool}" "${group}" "${image_prefix}" 'true' "${image_count}" times
+ results+=("image count:$image_count enable time:"${times[0]}" sync_time:"${times[1]})
+ done
+
+ for result in "${results[@]}"; do
+ echo -e "${RED}${result}${NO_COLOUR}"
+ done
+
+#results:
+#image count:0 enable time:0 sync_time:6
+#image count:5 enable time:4 sync_time:6
+#image count:10 enable time:9 sync_time:9
+#image count:15 enable time:15 sync_time:13
+#image count:20 enable time:20 sync_time:13
+#image count:25 enable time:25 sync_time:21
+#image count:30 enable time:30 sync_time:22
+
+}
+
+# create group with images then enable mirroring. Remove group without disabling mirroring
+declare -a test_create_group_with_image_remove_then_repeat_1=("${CLUSTER2}" "${CLUSTER1}" )
+
+test_create_group_with_image_remove_then_repeat_scenarios=1
+
+test_create_group_with_image_remove_then_repeat()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool="${pool0}"
+ local group="${group0}"
+
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" 1
+ group_create "${primary_cluster}" "${pool}/${group}"
+
+ local loop_instance
+ for loop_instance in $(seq 0 5); do
+ testlog "test_create_group_with_image_remove_then_repeat ${loop_instance}"
+
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}0"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 1
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 1
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 1
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}0"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 0
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ check_daemon_running "${secondary_cluster}"
+ done
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" 1
+}
+
+# create group with images then enable mirroring. Disable and re-enable repeatedly
+declare -a test_enable_disable_repeat_1=("${CLUSTER2}" "${CLUSTER1}" )
+
+test_enable_disable_repeat_scenarios=1
+
+test_enable_disable_repeat()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool="${pool0}"
+ local group="group"
+
+ local group_instance
+ for group_instance in $(seq 0 9); do
+ group_create "${primary_cluster}" "${pool}/${group}${group_instance}"
+ done
+
+ stop_mirrors "${primary_cluster}"
+ stop_mirrors "${secondary_cluster}"
+ start_mirrors "${secondary_cluster}"
+
+ local loop_instance
+ for loop_instance in $(seq 0 10); do
+ testlog "test_enable_disable_repeat ${loop_instance}"
+
+ for group_instance in $(seq 0 9); do
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}${group_instance}"
+ done
+
+ for group_instance in $(seq 0 9); do
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}${group_instance}"
+ done
+ done
+
+ check_daemon_running "${secondary_cluster}"
+
+ for group_instance in $(seq 0 9); do
+ group_remove "${primary_cluster}" "${pool}/${group}${group_instance}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}${group_instance}"
+ done
+}
+
+# add and remove images to/from a mirrored group
+declare -a test_mirrored_group_add_and_remove_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 5)
+
+test_mirrored_group_add_and_remove_images_scenarios=1
+
+test_mirrored_group_add_and_remove_images()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local group_image_count=$6
+
+ group_create "${primary_cluster}" "${pool}/${group}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0
+
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+ # create another image and populate it with some data
+ local image_name="test_image"
+ image_create "${primary_cluster}" "${pool}/${image_name}"
+ local io_count=10240
+ local io_size=4096
+ write_image "${primary_cluster}" "${pool}" "${image_name}" "${io_count}" "${io_size}"
+
+ # add, wait for stable and then remove the image from the group
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $((1+"${group_image_count}"))
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $((1+"${group_image_count}"))
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ fi
+
+ # re-add and immediately remove
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+
+ # check that expected number of images exist on secondary
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ # remove and immediately re-add a different image
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}2"
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}2"
+
+ # check that expected number of images exist on secondary
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ # remove all images from the group
+ group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ image_remove "${primary_cluster}" "${pool}/${image_name}"
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ # check that expected number of images exist on secondary - TODO this should be replaying, but deleting the last image seems to cause
+ # the group to go stopped atm
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ check_daemon_running "${secondary_cluster}"
+}
+
+# create group with images then enable mirroring. Remove all images from group and check state matches initial empty group state
+declare -a test_mirrored_group_remove_all_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 2)
+
+test_mirrored_group_remove_all_images_scenarios=1
+
+test_mirrored_group_remove_all_images()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local group_image_count=$6
+
+ group_create "${primary_cluster}" "${pool}/${group}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 0
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" 0
+
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+
+ # remove all images from the group
+ group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ # check that expected number of images exist on secondary
+ # TODO why is the state "stopped" - a new empty group is in the "replaying" state
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0
+
+ # adding the images back into the group causes it to go back to replaying
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ # remove all images from the group again
+ group_images_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
+ # check that expected number of images exist on secondary
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+stopped' 0
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ check_daemon_running "${secondary_cluster}"
}
+# create group then enable mirroring before adding images to the group. Disable mirroring before removing group
+declare -a test_create_group_mirror_then_add_images_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false')
+# create group then enable mirroring before adding images to the group. Remove group with mirroring enabled
+declare -a test_create_group_mirror_then_add_images_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true')
+
+test_create_group_mirror_then_add_images_scenarios=2
+
test_create_group_mirror_then_add_images()
{
local primary_cluster=$1
images_create "${primary_cluster}" "${pool}/${image_prefix}" 5
group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" 5
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' 0
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ fi
+
wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" 5
check_daemon_running "${secondary_cluster}"
check_daemon_running "${secondary_cluster}"
if [ 'false' != "${disable_before_remove}" ]; then
- mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
fi
group_remove "${primary_cluster}" "${pool}/${group}"
images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5
}
+#test remote namespace with different name
+declare -a test_remote_namespace_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}")
+
+test_remote_namespace_scenarios=1
+
+test_remote_namespace()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+
+ # configure primary namespace mirrored to secondary namespace with a different name
+ local primary_namespace
+ primary_namespace='ns3'
+ local secondary_namespace
+ secondary_namespace='ns3_remote'
+
+ local primary_pool_spec
+ local secondary_pool_spec
+ primary_pool_spec="${pool}/${primary_namespace}"
+ secondary_pool_spec="${pool}/${secondary_namespace}"
+
+ run_cmd "rbd --cluster ${primary_cluster} namespace create ${pool}/${primary_namespace}"
+ run_cmd "rbd --cluster ${secondary_cluster} namespace create ${pool}/${secondary_namespace}"
+
+ run_cmd "rbd --cluster ${primary_cluster} mirror pool enable ${pool}/${primary_namespace} image --remote-namespace ${secondary_namespace}"
+ run_cmd "rbd --cluster ${secondary_cluster} mirror pool enable ${pool}/${secondary_namespace} image --remote-namespace ${primary_namespace}"
+
+ group_create "${primary_cluster}" "${primary_pool_spec}/${group}"
+ mirror_group_enable "${primary_cluster}" "${primary_pool_spec}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" 0
+ wait_for_group_replay_started "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 0
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 0
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0
+ fi
+
+ try_cmd "rbd --cluster ${secondary_cluster} group snap list ${secondary_pool_spec}/${group}" || :
+ try_cmd "rbd --cluster ${primary_cluster} group snap list ${primary_pool_spec}/${group}" || :
+
+ mirror_group_disable "${primary_cluster}" "${primary_pool_spec}/${group}"
+
+ try_cmd "rbd --cluster ${secondary_cluster} group snap list ${secondary_pool_spec}/${group}" || :
+ try_cmd "rbd --cluster ${primary_cluster} group snap list ${primary_pool_spec}/${group}" || :
+
+ group_remove "${primary_cluster}" "${primary_pool_spec}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${primary_pool_spec}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}"
+ check_daemon_running "${secondary_cluster}"
+
+ # repeat the test - this time with some images
+ group_create "${primary_cluster}" "${primary_pool_spec}/${group}"
+ images_create "${primary_cluster}" "${primary_pool_spec}/${image_prefix}" 5
+ group_images_add "${primary_cluster}" "${primary_pool_spec}/${group}" "${primary_pool_spec}/${image_prefix}" 5
+
+ mirror_group_enable "${primary_cluster}" "${primary_pool_spec}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}" 5
+ wait_for_group_replay_started "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 5
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 5
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0
+ fi
+
+ # try a manual snapshot and check that it syncs
+ write_image "${primary_cluster}" "${primary_pool_spec}" "${image_prefix}0" 10 4096
+ local group_snap_id
+ mirror_group_snapshot "${primary_cluster}" "${primary_pool_spec}/${group}" group_snap_id
+ wait_for_group_snap_present "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}"
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}"
+
+ # Check all images in the group and confirms that they are synced
+ test_group_synced_image_status "${secondary_cluster}" "${secondary_pool_spec}/${group}" "${group_snap_id}" 5
+
+ # try demote, promote and resync
+ mirror_group_demote "${primary_cluster}" "${primary_pool_spec}/${group}"
+ wait_for_group_replay_stopped "${secondary_cluster}" "${secondary_pool_spec}/${group}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}/${group}" 'up+stopped' 0
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}/${group}" 'down+unknown' 0
+ mirror_group_promote "${secondary_cluster}" "${secondary_pool_spec}/${group}"
+
+ write_image "${secondary_cluster}" "${secondary_pool_spec}" "${image_prefix}0" 10 4096
+
+ mirror_group_demote "${secondary_cluster}" "${secondary_pool_spec}/${group}"
+ mirror_group_promote "${primary_cluster}" "${primary_pool_spec}/${group}"
+
+ mirror_group_resync "${secondary_cluster}" "${secondary_pool_spec}/${group}"
+
+ wait_for_group_synced "${primary_cluster}" "${primary_pool_spec}/${group}"
+
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${secondary_pool_spec}"/"${group}" 'up+replaying' 5
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${primary_pool_spec}"/"${group}" 'down+unknown' 0
+ fi
+
+ group_remove "${primary_cluster}" "${primary_pool_spec}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${primary_pool_spec}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${secondary_pool_spec}" "${group}"
+
+ images_remove "${primary_cluster}" "${primary_pool_spec}/${image_prefix}" 5
+ check_daemon_running "${secondary_cluster}"
+}
+
+#test empty group
+declare -a test_empty_group_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}")
+#test empty group with namespace
+declare -a test_empty_group_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}/${NS1}" "${group0}")
+
+test_empty_group_scenarios=2
+
test_empty_group()
{
local primary_cluster=$1
check_daemon_running "${secondary_cluster}"
}
+#check that omap keys have been correctly deleted
+declare -a test_empty_group_omap_keys_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}")
+
+test_empty_group_omap_keys_scenarios=1
+
+test_empty_group_omap_keys()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+
+ wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gremote_status"
+ wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global"
+ wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gremote_status"
+ wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global"
+
+ run_test test_empty_group 1
+
+ wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gremote_status"
+ wait_for_omap_keys "${primary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global"
+ wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gremote_status"
+ wait_for_omap_keys "${secondary_cluster}" "${pool}" "rbd_mirroring" "gstatus_global"
+}
+
+#check that pool doesn't get in strange state
+declare -a test_group_with_clone_image_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}")
+
+test_group_with_clone_image_scenarios=1
+
+test_group_with_clone_image()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+
+ group_create "${primary_cluster}" "${pool}/${group}"
+
+ # create an image and then clone it
+ image_create "${primary_cluster}" "${pool}/parent_image"
+ create_snapshot "${primary_cluster}" "${pool}" "parent_image" "snap1"
+ protect_snapshot "${primary_cluster}" "${pool}" "parent_image" "snap1"
+ clone_image "${primary_cluster}" "${pool}" "parent_image" "snap1" "${pool}" "child_image"
+
+ # create some other images
+ image_create "${primary_cluster}" "${pool}/other_image0"
+ image_create "${primary_cluster}" "${pool}/other_image1"
+
+ # add the other images and the clone to the group
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/other_image0"
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/other_image1"
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/child_image"
+
+ # next command fails with the following message
+ # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: image_enable: mirroring is not enabled for the parent
+ # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: group_enable: failed enabling image: child_image: (22) Invalid argument
+ mirror_group_enable_try "${primary_cluster}" "${pool}/${group}" || :
+ test 0 = "$(grep -c "interrupted" "$CMD_STDERR")" || fail "unexpected output"
+
+ # next command appears to succeed
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+
+ # another attempt at enable fails with a strange message
+ # 2025-01-30T17:17:34.421+0000 7f9ad0d74b40 -1 librbd::api::Mirror: group_enable: enabling mirroring for group test-group0 either in progress or was interrupted
+ mirror_group_enable_try "${primary_cluster}" "${pool}/${group}" || :
+ test 0 = "$(grep -c "interrupted" "$CMD_STDERR")" || fail "unexpected output"
+}
+
+test_from_nithya_that_will_stop_working_when_api_changes()
+{
+[root@server1 build]# rbd-a group create data/grp1
+[root@server1 build]# rbd-a group image add data/grp1 data/img-1
+[root@server1 build]# rbd-a group image add data/grp1 data/img-2
+[root@server1 build]# rbd-a group image add data/grp1 data/img-3
+[root@server1 build]# rbd-a mirror group enable data/grp1
+[root@server1 build]# rbd-a mirror image demote data/img-2
+[root@server1 build]# rbd-a mirror group snapshot data/grp1
+[root@server1 build]# rbd-a snap ls --all data/img-3
+[root@server1 build]# rbd-a group snap ls data/grp1
+}
+
+# test two empty groups
+declare -a test_empty_groups_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${group1}")
+
+test_empty_groups_scenarios=1
+
test_empty_groups()
{
local primary_cluster=$1
check_daemon_running "${secondary_cluster}"
}
+# add image from a different pool to group and test replay
+declare -a test_images_different_pools_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${pool1}" "${group0}" "${image_prefix}")
+
+test_images_different_pools_scenarios=1
+
# This test is not MVP
test_images_different_pools()
{
image_create "${primary_cluster}" "${pool1}/${image_prefix}1"
group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool1}/${image_prefix}1"
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ # check secondary cluster sees 0 images
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 0
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}"/"${group}"
+ fi
+
wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 2
wait_for_group_replay_started "${secondary_cluster}" "${pool0}"/"${group}" 2
wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 2
image_remove "${primary_cluster}" "${pool1}/${image_prefix}1"
}
+# create regular group snapshots and test replay
+declare -a test_create_group_with_images_then_mirror_with_regular_snapshots_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}")
+
+test_create_group_with_images_then_mirror_with_regular_snapshots_scenarios=1
+
test_create_group_with_images_then_mirror_with_regular_snapshots()
{
local primary_cluster=$1
local pool=$3
local group=$4
local image_prefix=$5
+ local snap='regular_snap'
group_create "${primary_cluster}" "${pool}/${group}"
images_create "${primary_cluster}" "${pool}/${image_prefix}" 5
wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
fi
- snap='regular_snap'
check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}"
check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}"
group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}"
check_group_snap_exists "${primary_cluster}" "${pool}/${group}" "${snap}"
# snap is currently copied to secondary cluster, where it remains in the "incomplete" state, but this is maybe incorrect - see slack thread TODO
+ # - should not be copied until mirrored.
mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+
check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}"
group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}"
images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5
}
-test_create_group_with_large_image()
-{
+# create regular group snapshots before enable mirroring
+declare -a test_create_group_with_regular_snapshots_then_mirror_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}")
+
+test_create_group_with_regular_snapshots_then_mirror_scenarios=1
+
+test_create_group_with_regular_snapshots_then_mirror()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local group_image_count=12
+ local snap='regular_snap'
+
+ group_create "${primary_cluster}" "${pool}/${group}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}"
+ check_group_snap_exists "${primary_cluster}" "${pool}/${group}" "${snap}"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}"
+# wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}"
+# wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ #if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+# wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+# fi
+
+ check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}"
+ # TODO this next command fails because the regular snapshot seems to get stuck in the "incomplete" state on the secondary
+ # and the mirror group snapshot (taken on mirror enable) never appears on the secondary.
+ wait_for_group_synced "${primary_cluster}" "${pool}/${group}"
+## mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+
+ group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}"
+ check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}"
+ # this next extra mirror_group_snapshot should not be needed - waiting for fix TODO
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}"
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+}
+
+# add a large image to group and test replay
+declare -a test_create_group_with_large_image_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}")
+
+test_create_group_with_large_image_scenarios=1
+
+test_create_group_with_large_image()
+{
local primary_cluster=$1
local secondary_cluster=$2
local pool0=$3
big_image=test-image-big
image_create "${primary_cluster}" "${pool0}/${big_image}" 4G
- group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
- write_image "${primary_cluster}" "${pool0}" "${big_image}" 1024 4194304
- wait_for_group_replay_started "${secondary_cluster}" "${pool0}/${group}" 2
- mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}"
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool0}/${group}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}/${group}" 'up+replaying' 2
+ fi
+ else
+ mirror_group_disable "${primary_cluster}" "${pool0}/${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool0}" "${group}"
+ group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
+ mirror_group_enable "${primary_cluster}" "${pool0}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 2
+ fi
- test_group_and_image_sync_status "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
+ write_image "${primary_cluster}" "${pool0}" "${big_image}" 1024 4194304
+ local group_snap_id
+ mirror_group_snapshot "${primary_cluster}" "${pool0}/${group}" group_snap_id
+ wait_for_group_snap_present "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}"
+
+ # TODO if the sync process could be controlled then we could check that test-image is synced before test-image-big
+ # and that the group is only marked as synced once both images have completed their sync
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}"
+
+ # Check all images in the group and confirms that they are synced
+ test_group_synced_image_status "${secondary_cluster}" "${pool0}/${group}" "${group_snap_id}" 2
+
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool0}"/"${group}" 'up+replaying' 1
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}"/"${group}"
+ fi
+ else
+ mirror_group_disable "${primary_cluster}" "${pool0}/${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool0}" "${group}"
+ group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
+ mirror_group_enable "${primary_cluster}" "${pool0}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${pool0}" "${group}" 1
+ fi
- group_image_remove "${primary_cluster}" "${pool0}/${group}" "${pool0}/${big_image}"
remove_image_retry "${primary_cluster}" "${pool0}" "${big_image}"
- wait_for_group_replay_started "${secondary_cluster}" "${pool0}/${group}" 1
mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool0}/${group}"
+ test_images_in_latest_synced_group "${secondary_cluster}" "${pool0}/${group}" 1
mirror_group_disable "${primary_cluster}" "${pool0}/${group}"
group_remove "${primary_cluster}" "${pool0}/${group}"
image_remove "${primary_cluster}" "${pool0}/${image_prefix}"
}
+# multiple images in group with io
+declare -a test_create_group_with_multiple_images_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}")
+
+test_create_group_with_multiple_images_do_io_scenarios=1
+
test_create_group_with_multiple_images_do_io()
{
local primary_cluster=$1
local io_count=1024
local io_size=4096
- write_image "${primary_cluster}" "${pool}" "${image_prefix}0" "${io_count}" "${io_size}"
+
+ local loop_instance
+ for loop_instance in $(seq 0 $((5-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}"
+ test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" 5
-exit 0
- # TODO this test needs finishing. The next function is not yet complete - see the TODO in it
- test_group_and_image_sync_status "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}" "${pool1}/${big_image}"
+ for loop_instance in $(seq 0 $((5-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}"
+ done
+ for loop_instance in $(seq 0 $((5-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
snap='regular_snap'
group_snap_create "${primary_cluster}" "${pool}/${group}" "${snap}"
mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
check_group_snap_exists "${secondary_cluster}" "${pool}/${group}" "${snap}"
+ test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" 5
+
+ for loop_instance in $(seq 0 $((5-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}"
+ done
+
+ group_snap_remove "${primary_cluster}" "${pool}/${group}" "${snap}"
+ check_group_snap_doesnt_exist "${primary_cluster}" "${pool}/${group}" "${snap}"
+ # this next extra mirror_group_snapshot should not be needed - waiting for fix TODO
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group}"
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}"/"${group}"
+ check_group_snap_doesnt_exist "${secondary_cluster}" "${pool}/${group}" "${snap}"
+
+ for loop_instance in $(seq 0 $((5-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}"
+ done
mirror_group_disable "${primary_cluster}" "${pool}/${group}"
group_remove "${primary_cluster}" "${pool}/${group}"
images_remove "${primary_cluster}" "${pool}/${image_prefix}" 5
}
-set -ex
+# multiple images in group with io
+declare -a test_stopped_daemon_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 3)
-# If the tmpdir or cluster conf file doesn't exist then assume that the cluster needs setting up
-if [ ! -d "${RBD_MIRROR_TEMDIR}" ] || [ ! -f "${RBD_MIRROR_TEMDIR}"'/cluster1.conf' ]
-then
- setup
-fi
-export RBD_MIRROR_USE_EXISTING_CLUSTER=1
+test_stopped_daemon_scenarios=1
+
+test_stopped_daemon()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local group_image_count=$6
-# rbd_mirror_helpers assumes that we are running from tmpdir
-setup_tempdir
+ check_daemon_running "${secondary_cluster}"
-# see if we need to (re)start rbd-mirror deamon
-pid=$(cat "$(daemon_pid_file "${CLUSTER1}")" 2>/dev/null) || :
-if [ -z "${pid}" ]
-then
- start_mirrors "${CLUSTER1}"
-fi
-check_daemon_running "${CLUSTER1}" 'restart'
+ group_create "${primary_cluster}" "${pool}/${group}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
-group0=test-group0
-group1=test-group1
-pool0=mirror
-pool1=mirror_parent
-image_prefix=test-image
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}"
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}"
+
+ local primary_group_snap_id
+ get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id
+ local secondary_group_snap_id
+ get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id
+ test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; }
+
+ # Add image to synced group (whilst daemon is stopped)
+ echo "stopping daemon"
+ stop_mirrors "${secondary_cluster}"
+
+ local image_name="test_image"
+ image_create "${primary_cluster}" "${pool}/${image_name}"
+
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool}/${group}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}/${group}" 'up+replaying' $(("${group_image_count}"+1))
+ fi
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ # wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+ group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ # wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" $(("${group_image_count}"+1))
+ fi
+
+ get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id
+ test "${primary_group_snap_id}" != "${secondary_group_snap_id}" || { fail "matched ids"; return 1; }
+
+ echo "starting daemon"
+ start_mirrors "${secondary_cluster}"
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" $(("${group_image_count}"+1))
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' $(("${group_image_count}"+1))
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}"
+
+ get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id
+ test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; }
+
+ # removed image from synced group (whilst daemon is stopped)
+ echo "stopping daemon"
+ stop_mirrors "${secondary_cluster}"
+
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+
+ if [ -n "${RBD_MIRROR_NEW_IMPLICIT_BEHAVIOUR}" ]; then
+ mirror_group_snapshot_and_wait_for_sync_complete "${primary_cluster}" "${secondary_cluster}" "${pool}/${group}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}/${group}" 'up+replaying' $(("${group_image_count}"))
+ fi
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+# wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+ group_image_remove "${primary_cluster}" "${pool}/${group}" "${pool}/${image_name}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+ # wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" $(("${group_image_count}"))
+ fi
+
+ get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}"/"${group}" primary_group_snap_id
+ test "${primary_group_snap_id}" != "${secondary_group_snap_id}" || { fail "matched ids"; return 1; }
+
+ echo "starting daemon"
+ start_mirrors "${secondary_cluster}"
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+ # TODO next command fails because rbd group snap list command fails with -2
+ # though group does exist on secondary
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group}"
+
+ get_newest_group_mirror_snapshot_id "${secondary_cluster}" "${pool}"/"${group}" secondary_group_snap_id
+ test "${primary_group_snap_id}" = "${secondary_group_snap_id}" || { fail "mismatched ids"; return 1; }
+
+ # TODO test more actions whilst daemon is stopped
+ # add image, take snapshot, remove image, take snapshot, restart
+ # disable mirroring
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+}
+
+# multiple images in group and standalone images too with io
+declare -a test_group_and_standalone_images_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false')
+declare -a test_group_and_standalone_images_do_io_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true')
+
+# TODO scenario 2 fails currently
+test_group_and_standalone_images_do_io_scenarios=1
+
+test_group_and_standalone_images_do_io()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group=$4
+ local image_prefix=$5
+ local test_pool_status=$6
+
+ local standalone_image_prefix=standalone-image
+ local standalone_image_count=4
+ local group_image_count=2
+
+ images_create "${primary_cluster}" "${pool}/${standalone_image_prefix}" "${standalone_image_count}"
+
+ group_create "${primary_cluster}" "${pool}/${group}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group}" "${pool}/${image_prefix}" "${group_image_count}"
+
+ if [ 'true' = "${test_pool_status}" ]; then
+ local fields=(//status/images/image/name //status/groups/group/name)
+ local pool_fields_count_arr=()
+ count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}"
+ # Check count of images and groups in the command output
+ echo "kk ${pool_fields_count_arr[0]} ${pool_fields_count_arr[1]}"
+ test 0 = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}"
+ test 0 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}"
+ fi
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group}"
+
+ if [ 'true' = "${test_pool_status}" ]; then
+ local fields=(//status/images/image/name //status/groups/group/name)
+ pool_fields_count_arr=()
+ count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}"
+ # Check count of images and groups in the command output
+ test $((${group_image_count})) = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}"
+ test 1 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}"
+ fi
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group}" "${group_image_count}"
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group}" "${group_image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group}" 'up+replaying' "${group_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group}" 'down+unknown' 0
+ fi
+
+ # enable mirroring for standalone images
+ local loop_instance
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ enable_mirror "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ wait_for_image_replay_started "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ wait_for_replay_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ wait_for_replaying_status_in_pool_dir "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ compare_images "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ done
+
+ if [ 'true' = "${test_pool_status}" ]; then
+ local fields=(//status/images/image/name //status/groups/group/name)
+ pool_fields_count_arr=()
+ count_fields_in_mirror_pool_status "${primary_cluster}" "${pool}" pool_fields_count_arr "${fields[@]}"
+ # Check count of images and groups in the command output
+ test $((${standalone_image_count}+${group_image_count})) = "${pool_fields_count_arr[0]}" || fail "unexpected count of images : ${pool_fields_count_arr[0]}"
+ test 1 = "${pool_fields_count_arr[1]}" || fail "unexpected count of groups : ${pool_fields_count_arr[1]}"
+ fi
+
+ local io_count=1024
+ local io_size=4096
+
+ # write to all of the images
+ for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+
+ # snapshot the group
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group}"
+ test_images_in_latest_synced_group "${secondary_cluster}" "${pool}/${group}" "${group_image_count}"
+
+ for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}"
+ done
+
+ # snapshot the individual images too, wait for sync and compare
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ mirror_image_snapshot "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ wait_for_snapshot_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ done
+
+ # do more IO
+ for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+
+ # Snapshot the group and images. Sync both in parallel
+ local group_snap_id
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" group_snap_id
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ mirror_image_snapshot "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ done
+
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}"
+ for loop_instance in $(seq 0 $(("${group_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${loop_instance}"
+ done
+
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ wait_for_snapshot_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ done
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group}"
+ group_remove "${primary_cluster}" "${pool}/${group}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group}"
+
+ # re-check images
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ done
+
+ # disable mirroring for standalone images
+ local loop_instance
+ for loop_instance in $(seq 0 $(("${standalone_image_count}"-1))); do
+ disable_mirror "${primary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}"
+ wait_for_image_present "${secondary_cluster}" "${pool}" "${standalone_image_prefix}${loop_instance}" 'deleted'
+ done
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${group_image_count}"
+ images_remove "${primary_cluster}" "${pool}/${standalone_image_prefix}" "${standalone_image_count}"
+}
+
+# multiple groups with images in each with io
+# mismatched size groups (same size images)
+declare -a test_create_multiple_groups_do_io_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 128 5 2 128)
+declare -a test_create_multiple_groups_do_io_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 128 1 1 128)
+# mismatched size groups (mismatched size images)
+declare -a test_create_multiple_groups_do_io_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 4 5 2 128)
+declare -a test_create_multiple_groups_do_io_4=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 1 10 4 1 1 128)
+# equal size groups
+declare -a test_create_multiple_groups_do_io_5=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 2 5 128 2 5 128)
+
+test_create_multiple_groups_do_io_scenarios=5
+
+test_create_multiple_groups_do_io()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local group_a_count=$4
+ local group_a_image_count=$5
+ local group_a_image_size=$6
+ local group_b_count=$7
+ local group_b_image_count=$8
+ local group_b_image_size=$9
+
+ local image_count=$(("${group_a_count}"*"${group_a_image_count}" + "${group_b_count}"*"${group_b_image_count}"))
+ local image_prefix='test-image'
+ local group_prefix='test-group'
+ local loop_instance
+ local group_instance
+
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ group_create "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}" "${group_a_image_size}"
+ group_images_add "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ group_create "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}" "${group_b_image_size}"
+ group_images_add "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}"
+ done
+
+ # enable mirroring for every group
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ mirror_group_enable "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ mirror_group_enable "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ done
+
+ # check that every group appears on the secondary
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}" "${group_a_image_count}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}" "${group_b_image_count}"
+ done
+
+ # export RBD_MIRROR_INSTANCES=X and rerun the test to check assignment
+ # image size appears not to influence the distribution of group replayers
+ # number of images in a group does influence the distribution
+ # FUTURE - implement some checking on the assignment rather than just printing it
+ # - also maybe change image count in groups and check rebalancing
+ local group_arr
+ local image_arr
+ for instance in $(seq 0 ${LAST_MIRROR_INSTANCE}); do
+ local result
+ query_replayer_assignment "${secondary_cluster}" "${instance}" result
+ group_arr+=("${result[0]}")
+ image_arr+=("${result[1]}")
+ group_count_arr+=("${result[2]}")
+ image_count_arr+=("${result[3]}")
+ done
+ for instance in $(seq 0 ${LAST_MIRROR_INSTANCE}); do
+ echo -e "${RED}MIRROR DAEMON INSTANCE:${instance}${NO_COLOUR}";
+ echo -e "${RED}GROUP_REPLAYERS:${group_count_arr[$instance]}${NO_COLOUR}";
+ echo -e "${RED}${group_arr[$instance]}${NO_COLOUR}";
+ echo -e "${RED}IMAGE_REPLAYERS:${image_count_arr[$instance]}${NO_COLOUR}";
+ echo -e "${RED}${image_arr[$instance]}${NO_COLOUR}";
+ done
+
+ # check that every group and image are in the correct state on the secondary
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" "${group_a_image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" 'up+replaying' "${group_a_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group_prefix}-a${loop_instance}" 'down+unknown' 0
+ fi
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" "${group_b_image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" 'up+replaying' "${group_b_image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group_prefix}-b${loop_instance}" 'down+unknown' 0
+ fi
+ done
+
+ local io_count=10240
+ local io_size=4096
+ local group_to_mirror='-a0'
+
+ # write to every image in one a group, mirror group and compare images
+ for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" "${io_count}" "${io_size}"
+ done
+
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}${group_to_mirror}"
+
+ for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}"
+ done
+
+ group_to_mirror='-b0'
+
+ # write to every image in one b group, mirror group and compare images
+ for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}" "${io_count}" "${io_size}"
+ done
+
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}${group_to_mirror}"
+
+ for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}${group_to_mirror}-${loop_instance}"
+ done
+
+ # write to one image in every group, mirror groups and compare images
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}-a${loop_instance}-0" "${io_count}" "${io_size}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}-b${loop_instance}-0" "${io_count}" "${io_size}"
+ done
+
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ done
+
+ for group_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}"
+ done
+ done
+ for group_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}"
+ done
+ done
+
+ # write to every image in every group, mirror groups and compare images
+ for group_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}" "${io_count}" "${io_size}"
+ done
+ done
+ for group_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}" "${io_count}" "${io_size}"
+ done
+ done
+
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ done
+
+ for group_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_a_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-a${group_instance}-${loop_instance}"
+ done
+ done
+ for group_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ for loop_instance in $(seq 0 $(("${group_b_image_count}"-1))); do
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}-b${group_instance}-${loop_instance}"
+ done
+ done
+
+ # disable and remove all groups
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ mirror_group_disable "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ group_remove "${primary_cluster}" "${pool}/${group_prefix}-a${loop_instance}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ mirror_group_disable "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ group_remove "${primary_cluster}" "${pool}/${group_prefix}-b${loop_instance}"
+ done
+
+ # check all groups have been deleted and remove images
+ for loop_instance in $(seq 0 $(("${group_a_count}"-1))); do
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group_prefix}-a${loop_instance}"
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}-a${loop_instance}-" "${group_a_image_count}"
+ done
+ for loop_instance in $(seq 0 $(("${group_b_count}"-1))); do
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group_prefix}-b${loop_instance}"
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}-b${loop_instance}-" "${group_b_image_count}"
+ done
+}
+
+# mirror a group then remove an image from that group and add to a different mirrored group.
+declare -a test_image_move_group_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}")
+
+test_image_move_group_scenarios=1
+
+test_image_move_group()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local image_prefix=$4
+
+ local image_count=5
+ local group0=test-group0
+ local group1=test-group1
+
+ group_create "${primary_cluster}" "${pool}/${group0}"
+ group_create "${primary_cluster}" "${pool}/${group1}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+ group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" "${image_count}"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
-testlog "TEST: empty group"
-test_empty_group "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}"
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
-testlog "TEST: empty group with namespace"
-test_empty_group "${CLUSTER2}" "${CLUSTER1}" "${pool0}/${NS1}" "${group0}"
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0
+ fi
-testlog "TEST: create group with images then enable mirroring. Remove group without disabling mirroring"
-test_create_group_with_images_then_mirror "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false'
+ local io_count=10240
+ local io_size=4096
-testlog "TEST: create group with images then enable mirroring. Disable mirroring then remove group"
-test_create_group_with_images_then_mirror "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true'
+ # write to every image in the group, mirror group
+ for loop_instance in $(seq 0 $(("${image_count}"-1))); do
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}${loop_instance}" "${io_count}" "${io_size}"
+ done
+ mirror_group_snapshot_and_wait_for_sync_complete "${secondary_cluster}" "${primary_cluster}" "${pool}/${group0}"
+
+ # remove an image from the group and add to a different group
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}4"
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}4"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ fi
-testlog "TEST: create group then enable mirroring before adding images to the group. Remove group without disabling mirroring"
-test_create_group_mirror_then_add_images "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'false'
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}4"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group1}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 1
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 1
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 1
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-1))
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-1))
+
+ # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled)
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}2"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}2"
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ mirror_group_disable "${primary_cluster}" "${pool}/${group1}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group1}"
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}2"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}2"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group1}"
+ fi
-testlog "TEST: create group then enable mirroring before adding images to the group. Disable mirroring then remove group"
-test_create_group_mirror_then_add_images "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}" 'true'
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 2
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 2
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 2
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-2))
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-2))
+
+ echo "stopping daemon"
+ stop_mirrors "${secondary_cluster}"
+
+ # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled) with the mirror daemon stopped
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}0"
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ mirror_group_disable "${primary_cluster}" "${pool}/${group1}"
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}0"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group1}"
+ fi
-testlog "TEST: two empty groups"
-test_empty_groups "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${group1}"
+ echo "starting daemon"
+ start_mirrors "${secondary_cluster}"
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 3
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 3
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 3
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-3))
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-3))
+
+ echo "stopping daemon"
+ stop_mirrors "${secondary_cluster}"
+
+ # remove another image from group0 - add to group 1 (add to a group that is already mirror enabled) with the mirror daemon stopped
+ # this time the moved image is still present in a snapshot for the old group that needs syncing and in a snapshot for the new group
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+
+ if [ -n "${RBD_MIRROR_SUPPORT_DYNAMIC_GROUPS}" ]; then
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}1"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}1"
+ else
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ mirror_group_disable "${primary_cluster}" "${pool}/${group1}"
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}1"
+ group_image_add "${primary_cluster}" "${pool}/${group1}" "${pool}/${image_prefix}1"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ mirror_group_enable "${primary_cluster}" "${pool}/${group1}"
+ fi
-# testlog "TEST: add image from a different pool to group and test replay"
-# different pools - not MVP
-# test_images_different_pools "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${pool1}" "${group0}" "${image_prefix}"
+ echo "starting daemon"
+ start_mirrors "${secondary_cluster}"
-testlog "TEST: create regular group snapshots and test replay"
-test_create_group_with_images_then_mirror_with_regular_snapshots "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group1}" 4
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group1}" 4
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group1}" 'up+replaying' 4
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" $(("${image_count}"-4))
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' $(("${image_count}"-4))
-testlog "TEST: add a large image to group and test replay"
-test_create_group_with_large_image "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}"
+ # set up a chain of moves TODO
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ group_remove "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group1}"
+ group_remove "${primary_cluster}" "${pool}/${group1}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group1}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group1}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
-#TODO - test with mirrored images not in group
+# test force promote scenarios
+# TODO first two scenarios require support for dynamic groups
+#declare -a test_force_promote_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_add')
+#declare -a test_force_promote_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_remove')
+declare -a test_force_promote_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_rename')
+declare -a test_force_promote_2=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_expand')
+declare -a test_force_promote_3=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'image_shrink')
+declare -a test_force_promote_4=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'no_change')
+
+test_force_promote_scenarios=4
+
+test_force_promote()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local image_prefix=$4
+ local scenario=$5
+
+ local image_count=5
+ local group0=test-group0
+ local snap0='snap_0'
+ local snap1='snap_1'
+
+ group_create "${primary_cluster}" "${pool}/${group0}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" $(("${image_count}"-1))
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" $(("${image_count}"-1))
+ create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap0}"
+ compare_image_with_snapshot "${primary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ big_image=test-image-big
+ image_create "${primary_cluster}" "${pool}/${big_image}" 4G
+ group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${big_image}"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0
+ fi
+
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
+ compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap1}"
+
+ # make some changes to the big image so that the next sync will take a long time
+ write_image "${primary_cluster}" "${pool}" "${big_image}" 1024 4194304
+
+ local global_id
+ local image_size
+ local test_image_size
+ if [ "${scenario}" = 'image_add' ]; then
+ new_image=test-image-new
+ image_create "${primary_cluster}" "${pool}/${new_image}"
+ group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${new_image}"
+ get_image_mirroring_global_id "${primary_cluster}" "${pool}/${new_image}" global_id
+ test_image_with_global_id_present "${primary_cluster}" "${pool}" "${new_image}" "${global_id}"
+ test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${new_image}" "${global_id}"
+ elif [ "${scenario}" = 'image_remove' ]; then
+ get_image_mirroring_global_id "${primary_cluster}" "${pool}/${image_prefix}0" global_id
+ test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0"
+ test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ elif [ "${scenario}" = 'image_rename' ]; then
+ get_image_mirroring_global_id "${primary_cluster}" "${pool}/${image_prefix}0" global_id
+ image_rename "${primary_cluster}" "${pool}/${image_prefix}0" "${pool}/${image_prefix}_renamed_0"
+ test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}"
+ test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}"
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+ elif [ "${scenario}" = 'image_expand' ]; then
+ get_image_size "${primary_cluster}" "${pool}/${image_prefix}2" image_size
+ image_resize "${primary_cluster}" "${pool}/${image_prefix}2" $((("${image_size}"/1024/1024)+4))
+ test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}2" $(("${image_size}"+4*1024*1024))
+ test_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}2" "${image_size}"
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+ elif [ "${scenario}" = 'image_shrink' ]; then
+ get_image_size "${primary_cluster}" "${pool}/${image_prefix}3" image_size
+ image_resize "${primary_cluster}" "${pool}/${image_prefix}3" $((("${image_size}"/1024/1024)-4)) '--allow-shrink'
+ test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}3" $(("${image_size}"-4*1024*1024))
+ test_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}3" "${image_size}"
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+ elif [ "${scenario}" = 'no_change' ]; then
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+ fi
-: ' # TODO next test needs finishing
-testlog "TEST: multiple images in group with io"
-test_create_group_with_multiple_images_do_io "${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${group0}" "${image_prefix}"
+ # TODO add the following test
+: '
+ # This test removes and recreates an image - it fails currently as the request to list the group snaps on the secondary fails
+ group_image_remove "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0"
+ image_remove "${primary_cluster}" "${pool}/${image_prefix}0"
+ image_create "${primary_cluster}" "${pool}/${image_prefix}0" maybe different size?
+ group_image_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}0"
'
+ local group_snap_id
+ get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}/${group0}" group_snap_id
+ echo "id = ${group_snap_id}"
+ wait_for_test_group_snap_present "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1
+
+ if [ "${scenario}" = 'image_add' ]; then
+ wait_for_image_present "${secondary_cluster}" "${pool}" "${new_image}" 'present'
+ test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${new_image}" "${global_id}"
+ elif [ "${scenario}" = 'image_remove' ]; then
+ wait_for_image_present "${secondary_cluster}" "${pool}" "${image_prefix}0" 'deleted'
+ test_image_with_global_id_not_present "${secondary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" $(("${image_count}"-1))
+ elif [ "${scenario}" = 'image_rename' ]; then
+ wait_for_image_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" 'present'
+ test_image_with_global_id_present "${secondary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}"
+ elif [ "${scenario}" = 'image_expand' ]; then
+ wait_for_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}2" $(("${image_size}"+4*1024*1024))
+ elif [ "${scenario}" = 'image_shrink' ]; then
+ wait_for_image_size_matches "${secondary_cluster}" "${pool}/${image_prefix}3" $(("${image_size}"-4*1024*1024))
+ fi
+
+ # stop the daemon to prevent further syncing of snapshots
+ stop_mirrors "${secondary_cluster}"
+
+ # check that latest snap is incomplete
+ ## this fails in the delete case as follows:
+ ##CEPH_ARGS='--id mirror' rbd --cluster cluster1 group snap list mirror/group_0
+ ##ERR: rc= 2
+ test_group_snap_sync_incomplete "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}"
+
+ # force promote the group on the secondary - should rollback to the last complete snapshot
+ local old_primary_cluster
+ mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force'
+ old_primary_cluster="${primary_cluster}"
+ primary_cluster="${secondary_cluster}"
+
+ mirror_group_demote "${old_primary_cluster}" "${pool}/${group0}"
+ secondary_cluster="${old_primary_cluster}"
+
+ # Check that the rollback reverted the state
+ if [ "${scenario}" = 'image_add' ]; then
+ # check that new image is not present
+ test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${new_image}" "${global_id}"
+ elif [ "${scenario}" = 'image_remove' ]; then
+ test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ elif [ "${scenario}" = 'image_rename' ]; then
+ # check that the image is back with the original name
+ test_image_with_global_id_not_present "${primary_cluster}" "${pool}" "${image_prefix}_renamed_0" "${global_id}"
+ test_image_with_global_id_present "${primary_cluster}" "${pool}" "${image_prefix}0" "${global_id}"
+ elif [ "${scenario}" = 'image_expand' ]; then
+ test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}2" "${image_size}" || fail "size mismatch"
+ elif [ "${scenario}" = 'image_shrink' ]; then
+ test_image_size_matches "${primary_cluster}" "${pool}/${image_prefix}3" "${image_size}" || fail "size mismatch"
+ fi
+
+
+ mirror_group_resync ${secondary_cluster} ${pool}/${group0}
+
+ start_mirrors "${secondary_cluster}"
+ sleep 5
+# TODO check that data can be copied back to original primary cluster
+# next line fails because latest snapshot on primary is never copied back to secondary
+# finish off the resync function
+# check that tidy up steps below work
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
+
+ compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ # Check that snapshots work on the new primary
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group}" group_snap_id
+ wait_for_group_snap_present "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}"
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool}/${group}" "${group_snap_id}"
+
+ # tidy up
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ group_remove "${primary_cluster}" "${pool}/${group0}"
+
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
+
+# test force promote scenarios
+declare -a test_multiple_user_snapshot_time_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}")
+
+test_multiple_user_snapshot_time_scenarios=1
+
+test_multiple_user_snapshot_time()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+
+ local image_count
+ local image_counts=(2 6)
+ local results=()
+ local time
+
+ for image_count in "${image_counts[@]}"; do
+ test_multiple_user_snapshot_whilst_stopped "${primary_cluster}" "${secondary_cluster}" "${pool}" "${image_count}" time
+ results+=(${time})
+ done
+
+ for i in "${#results[@]}"; do
+ echo -e "${RED}image count:"$image_counts[$i]" snapshot time:"${results[$i]}"${NO_COLOUR}"
+ done
+
+ if [ ${results[1]} -gt $((${results[0]}+2)) ]; then
+ fail "Snapshot time isn't independent of the group image count"
+ fi
+}
+
+# test force promote scenarios
+declare -a test_multiple_user_snapshot_whilst_stopped_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 5)
+
+test_multiple_user_snapshot_whilst_stopped_scenarios=1
+
+test_multiple_user_snapshot_whilst_stopped()
+{
+ local primary_cluster=$1 ; shift
+ local secondary_cluster=$1 ; shift
+ local pool=$1 ; shift
+ local image_count=$1 ; shift
+ if [ -n "$1" ]; then
+ local get_average='true'
+ local -n _average_snapshot_time=$1 ; shift
+ fi
+
+ local group0=test-group0
+ local image_prefix="test_image"
+
+ group_create "${primary_cluster}" "${pool}/${group0}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" 12M
+ group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" "${image_count}"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
+
+ echo "stopping daemon on secondary"
+ stop_mirrors "${secondary_cluster}"
+ # TODO starting the daemon on the primary seem to cause a problem with image deletion - Nithya investigating (see slack thread)
+ #echo "starting daemon on primary"
+ #start_mirrors "${primary_cluster}"
+
+ local start_time end_time
+ local times_result_arr=()
+ for i in {0..9}; do
+ start_time=$(date +%s)
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}"
+ end_time=$(date +%s)
+ echo -e "mirror group snapshot time ="$((end_time-start_time))
+ times_result_arr+=($((end_time-start_time)))
+ done;
+
+ if [ -n "$get_average" ]; then
+ local cnt total
+ total=0
+ cnt=0
+ # ignore the times of the first 5 snapshots then determine the average of the rest
+ for i in {5..9}; do
+ total=$((total + times_result_arr[$i]))
+ cnt=$((cnt + 1))
+ done;
+ echo "average=$((total/cnt))"
+ _average_snapshot_time=$((total/cnt))
+ fi
+
+ local count
+ get_group_snap_count "${primary_cluster}" "${pool}"/"${group0}" '*' count
+ test "${count}" -gt 3 || { fail "snap count = ${count}"; return 1; }
+
+ get_group_snap_count "${secondary_cluster}" "${pool}"/"${group0}" '*' count
+ test "${count}" -eq 1 || { fail "snap count = ${count}"; return 1; }
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+ echo "starting daemon on secondary"
+ start_mirrors "${secondary_cluster}"
+
+ # TODO remove this if when group successfully moves to synced state
+ if [ -z "$get_average" ]; then
+ # TODO this fails with the last image incomplete
+ wait_for_group_synced "${primary_cluster}" "${pool}/${group0}"
+
+ get_group_snap_count "${primary_cluster}" "${pool}"/"${group0}" '*' count
+ get_group_snap_count "${secondary_cluster}" "${pool}"/"${group0}" '*' count
+
+ # TODO this fails with the image on the secondary in split-brain
+ wait_for_status_in_pool_dir "${secondary_cluster}" "${pool}" "${image_prefix}" 'up+replaying'
+ fi
+
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+ wait_for_group_present "${primary_cluster}" "${pool}" "${group0}" "${image_count}"
+ #mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ group_remove "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
+
+# test resync scenarios
+declare -a test_resync_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" "${image_prefix}" 'no_change')
+
+test_resync_scenarios=1
+
+test_resync()
+{
+ local primary_cluster=$1
+ local secondary_cluster=$2
+ local pool=$3
+ local image_prefix=$4
+ local scenario=$5
+
+ local image_count=5
+ local group0=test-group0
+ local snap0='snap_0'
+ local snap1='snap_1'
+
+ group_create "${primary_cluster}" "${pool}/${group0}"
+ images_create "${primary_cluster}" "${pool}/${image_prefix}" $(("${image_count}"))
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" $(("${image_count}"))
+
+ create_snapshot "${primary_cluster}" "${pool}" "${image_prefix}0" "${snap0}"
+ compare_image_with_snapshot "${primary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+ wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+ wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}"
+ wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
+
+ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'down+unknown' 0
+ fi
+
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
+ compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ local group_snap_id secondary_group_snap_id primary_group_snap_id
+ get_newest_group_mirror_snapshot_id "${primary_cluster}" "${pool}/${group0}" primary_group_snap_id
+ echo "id = ${primary_group_snap_id}"
+
+ # stop the daemon to prevent further syncing of snapshots
+ stop_mirrors "${secondary_cluster}"
+
+ # promote secondary and change data on image
+ mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force'
+ write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ # demote secondary again
+ mirror_group_demote "${secondary_cluster}" "${pool}/${group0}"
+
+ # restart daemon and request a resync from primary
+ start_mirrors "${secondary_cluster}"
+ mirror_group_resync ${secondary_cluster} ${pool}/${group0}
+
+ # confirm that data on secondary again matches initial snapshot on primary
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
+ compare_image_with_snapshot "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ # Repeat the test this time changing the data on the primary too.
+
+ # stop the daemon to prevent further syncing of snapshots
+ stop_mirrors "${secondary_cluster}"
+
+ # promote secondary and change data on image
+ mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force'
+ write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" group_snap_id
+
+ # demote secondary again
+ mirror_group_demote "${secondary_cluster}" "${pool}/${group0}"
+
+ # restart daemon and request a resync from primary
+ start_mirrors "${secondary_cluster}"
+ mirror_group_resync ${secondary_cluster} ${pool}/${group0}
+
+ # confirm that data on secondary again matches latest snapshot on primary
+ wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
+ wait_for_test_group_snap_present "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}0"
+
+ # Repeat the test this time swapping the primary and secondary and resyncing back to the new secondary.
+: '
+ # TODO this test fails - does not sync back to old primary
+
+ # stop the daemon to prevent further syncing of snapshots
+ stop_mirrors "${secondary_cluster}"
+
+ # promote secondary and change data on image
+ mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force'
+ write_image "${secondary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+
+ compare_image_with_snapshot_expect_difference "${secondary_cluster}" "${pool}/${image_prefix}0" "${primary_cluster}" "${pool}/${image_prefix}0@${snap0}"
+
+ # change data on old primary too
+ write_image "${primary_cluster}" "${pool}" "${image_prefix}0" 10 4096
+ mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" group_snap_id
+
+ # demote old primary
+ mirror_group_demote "${primary_cluster}" "${pool}/${group0}"
+
+ # restart daemon and request a resync from primary
+ start_mirrors "${primary_cluster}"
+ mirror_group_resync ${primary_cluster} ${pool}/${group0}
+
+ # confirm that data on secondary again matches latest snapshot on primary
+ wait_for_group_synced "${secondary_cluster}" "${pool}"/"${group0}"
+ wait_for_test_group_snap_present "${primary_cluster}" "${pool}/${group0}" "${group_snap_id}" 1
+ compare_images "${primary_cluster}" "${secondary_cluster}" "${pool}" "${pool}" "${image_prefix}0"
+
+'
+
+ mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+ group_remove "${primary_cluster}" "${pool}/${group0}"
+
+ wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}"
+ wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+
+ images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
+
+run_test()
+{
+ local test_name=$1
+ local test_scenario=$2
+
+ declare -n test_parameters="$test_name"_"$test_scenario"
+
+ testlog "TEST:$test_name scenario:$test_scenario parameters:" "${test_parameters[@]}"
+ "$test_name" "${test_parameters[@]}"
+}
+
+# exercise all scenarios that are defined for the specified test
+run_test_scenarios()
+{
+ local test_name=$1
+
+ declare -n test_scenario_count="$test_name"_scenarios
+
+ local loop
+ for loop in $(seq 1 $test_scenario_count); do
+ run_test $test_name $loop
+ done
+}
+
+# exercise all scenarios for all tests
+run_tests()
+{
+ run_test_scenarios test_empty_group
+ run_test_scenarios test_empty_groups
+ # This next test requires support for dynamic groups TODO
+ # run_test_scenarios test_mirrored_group_remove_all_images
+ # This next test is unreliable - image/group ends up in stopped also requires dynamic groups - TODO enable
+ # run_test_scenarios test_mirrored_group_add_and_remove_images
+ # This next test is unreliable - image ends up in stopped also requires dynamic groups - TODO enable
+ # run_test_scenarios test_create_group_mirror_then_add_images
+ run_test_scenarios test_create_group_with_images_then_mirror
+ # next test is not MVP - TODO
+ # run_test_scenarios test_images_different_pools
+ # TODO next test fails if run with other tests - seems to have passed on its own must retry
+ # run_test_scenarios test_create_group_with_images_then_mirror_with_regular_snapshots
+ run_test_scenarios test_create_group_with_large_image
+ #run_test_scenarios test_create_group_with_multiple_images_do_io
+ run_test_scenarios test_group_and_standalone_images_do_io
+ run_test_scenarios test_create_multiple_groups_do_io
+ #run_test_scenarios test_stopped_daemon
+ #run_test_scenarios test_create_group_with_regular_snapshots_then_mirror
+ #run_test_scenarios test_image_move_group
+ #run_test_scenarios test_force_promote
+ #run_test_scenarios test_resync
+ run_test_scenarios test_remote_namespace
+ #run_test_scenarios test_multiple_user_snapshot_whilst_stopped
+ #run_test_scenarios test_create_group_with_image_remove_then_repeat
+ #run_test_scenarios test_enable_disable_repeat
+ #run_test_scenarios test_empty_group_omap_keys
+ #run_test_scenarios test_group_with_clone_image
+ #run_test_scenarios test_multiple_user_snapshot_time
+}
+
+if [ -n "${RBD_MIRROR_SHOW_CMD}" ]; then
+ set -e
+else
+ set -ex
+fi
+
+# If the tmpdir and cluster conf file exist then reuse the existing cluster
+if [ -d "${RBD_MIRROR_TEMDIR}" ] && [ -f "${RBD_MIRROR_TEMDIR}"'/cluster1.conf' ]
+then
+ export RBD_MIRROR_USE_EXISTING_CLUSTER=1
+fi
+
+setup
+
+# see if we need to (re)start rbd-mirror deamon
+pid=$(cat "$(daemon_pid_file "${CLUSTER1}")" 2>/dev/null) || :
+if [ -z "${pid}" ]
+then
+ start_mirrors "${CLUSTER1}"
+fi
+check_daemon_running "${CLUSTER1}"
+
+# restore the arguments from the cli
+set -- "${args[@]}"
+
+# loop count is specified as first argument. default value is 1
+loop_count="${1:-1}"
+for loop in $(seq 1 "${loop_count}"); do
+ echo "run number ${loop} of ${loop_count}"
+ if [ "$#" -gt 2 ]
+ then
+ # second arg is test_name
+ # third arg is scenario number
+ # fourth arg defines RBD_IMAGE_FEATURES (see top of file)
+ run_test "$2" "$3"
+ else
+ run_tests
+ fi
+done
+
exit 0
fi
RED='\033[0;31m'
+GREEN='\033[0;32m'
NO_COLOUR='\033[0m'
export CEPH_ARGS="--id ${CEPH_ID}"
fi
if [ -n "${fatal}" ]; then
- echo "${fatal}" 1>&2
+ echo -e "${RED}${fatal}" 1>&2
print_stacktrace
+ echo -e "${NO_COLOUR}"
exit 1
fi
local admin_key_file
local uuid
+ # Create and delete a random number of pools, images and snapshots so that ids on the two clusters sometimes mismatch
+ pool_count=$(( RANDOM % 2 ))
+ for loop_instance in $(seq 0 ${pool_count}); do
+ run_admin_cmd "ceph --cluster ${cluster} osd pool create dummy_pool 64 64"
+ image_create "${cluster}" "dummy_pool/dummy_image"
+ create_snapshot "${cluster}" "dummy_pool" "dummy_image" "dummy_snap"
+ group_create "${cluster}" "dummy_pool/dummy_group"
+ group_snap_create "${cluster}" "dummy_pool/dummy_group" "dummy_snap"
+ run_admin_cmd "ceph --cluster ${cluster} osd pool delete dummy_pool dummy_pool --yes-i-really-really-mean-it"
+ done
+
CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64
CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64
local error_code=$1
set +e
-
- if [ "${error_code}" -ne 0 ]; then
+ if [ "${error_code}" -ne 0 ] && [ -z "${RBD_MIRROR_NO_STATUS}" ]; then
status
fi
set_cluster_instance "${cluster}" cluster instance
test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
- local log=${TEMPDIR}/rbd-mirror${instance}.out
+ local log=${TEMPDIR}/rbd-mirror-${cluster}-${instance}.out
ulimit -c unlimited
rbd-mirror \
local status_log=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.status)
local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.status)
- mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}"
get_newest_mirror_snapshot "${cluster}" "${remote_pool}" "${image}" "${status_log}"
local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log})
if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
wait_for_journal_replay_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image}
elif [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+ mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}"
wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image}
else
return 1
fi
}
+count_fields_in_mirror_pool_status()
+{
+ local cluster=$1 ; shift
+ local pool=$1 ; shift
+ local -n _pool_result_count_arr=$1 ; shift
+ local fields=("$@")
+
+ run_cmd "rbd --cluster ${cluster} mirror pool status --verbose ${pool} --format xml --pretty-format" || { fail; return 1; }
+
+ local field result
+ for field in "${fields[@]}"; do
+ result=$($XMLSTARLET sel -t -v "count($field)" < "$CMD_STDOUT")
+ _pool_result_count_arr+=( "${result}" )
+ done
+}
+
+get_fields_from_mirror_pool_status()
+{
+ local cluster=$1 ; shift
+ local pool=$1 ; shift
+ local -n _pool_result_arr=$1 ; shift
+ local fields=("$@")
+
+ run_cmd "rbd --cluster ${cluster} mirror pool status --verbose ${pool} --format xml --pretty-format" || { fail; return 1; }
+
+ local field result
+ for field in "${fields[@]}"; do
+ result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; }
+ _pool_result_arr+=( "${result}" )
+ done
+}
+
+get_fields_from_mirror_group_status()
+{
+ local cluster=$1 ; shift
+ local group_spec=$1 ; shift
+ local -n _group_result_arr=$1 ; shift
+ local fields=("$@")
+
+ run_admin_cmd "rbd --cluster ${cluster} mirror group status ${group_spec} --format xml --pretty-format" || { fail; return 1; }
+
+ local field result
+ for field in "${fields[@]}"; do
+ result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; }
+ _group_result_arr+=( "${result}" )
+ done
+}
+
+get_fields_from_mirror_image_status()
+{
+ local cluster=$1 ; shift
+ local image_spec=$1 ; shift
+ local -n _image_result_arr=$1 ; shift
+ local fields=("$@")
+
+ run_admin_cmd "rbd --cluster ${cluster} mirror image status ${image_spec} --format xml --pretty-format" || { fail; return 1; }
+
+ local field result
+ for field in "${fields[@]}"; do
+ result=$($XMLSTARLET sel -t -v "$field" < "$CMD_STDOUT") || { fail "field not found: ${field}"; return; }
+ _image_result_arr+=( "${result}" )
+ done
+}
+
+check_fields_in_group_and_image_status()
+{
+ local cluster=$1
+ local group_spec=$2
+
+ local fields=(//group/state //group/description)
+ local group_fields_arr
+ get_fields_from_mirror_group_status "${cluster}" "${group_spec}" group_fields_arr "${fields[@]}"
+
+ local image_spec
+ for image_spec in $(rbd --cluster "${cluster}" group image list "${group_spec}" | xargs); do
+ local fields=(//image/state //image/description)
+ local image_fields_arr
+ get_fields_from_mirror_image_status "${cluster}" "${image_spec}" image_fields_arr "${fields[@]}"
+
+ # check that the image "state" matches the group "state"
+# TODO. The imaage status doesn not always get updated before the group status - see slack thread. Fail and allow retry for now
+# test "${image_fields_arr[0]}" = "${group_fields_arr[0]}" || { fail "image:${image_spec} ${image_fields_arr[0]} != ${group_fields_arr[0]}"; return 1; }
+ test "${image_fields_arr[0]}" = "${group_fields_arr[0]}" || { fail; return 1; }
+
+ # check that the image "description" matches the group "description". Need to remove the extra information from the image description first
+ local image_description
+ image_description=$(cut -d ',' -f 1 <<< "${image_fields_arr[1]}")
+# test "${image_description}" = "${group_fields_arr[1]}" || { fail "image:${image_spec} ${image_description} != ${group_fields_arr[1]}"; return 1; }
+ test "${image_description}" = "${group_fields_arr[1]}" || { fail; return 1; }
+ done
+}
test_status_in_pool_dir()
{
rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name}
}
+image_rename()
+{
+ local cluster=$1
+ local src_image_spec=$2
+ local dst_image_spec=$3
+
+ run_cmd "rbd --cluster=${cluster} rename ${src_image_spec} ${dst_image_spec}"
+}
+
remove_image()
{
local cluster=$1
return ${ret}
}
+compare_image_with_snapshot()
+{
+ local img_cluster=$1 ; shift
+ local image_spec=$1 ; shift
+ local snap_cluster=$1 ; shift
+ local snap_spec=$1 ; shift
+
+ if [ -n "$1" ]; then
+ expect_difference=$1 ; shift
+ fi
+
+ local ret=0
+
+ local img_export snap_export
+ img_export=${TEMPDIR}/$(mkfname ${img_cluster}-${image_spec}.export)
+ snap_export=${TEMPDIR}/$(mkfname ${snap_cluster}-${snap_spec}.export)
+ rm -f "${img_export}" "${snap_export}"
+
+ rbd --cluster "${img_cluster}" export "${image_spec}" "${img_export}"
+ rbd --cluster "${snap_cluster}" export "${snap_spec}" "${snap_export}"
+
+ if ! cmp "${img_export}" "${snap_export}"
+ then
+ if [ 'true' != "${expect_difference}" ]; then
+ show_diff "${img_export}" "${snap_export}"
+ ret=1
+ fi
+ fi
+ rm -f "${img_export}" "${snap_export}"
+ return "${ret}"
+}
+
+compare_image_with_snapshot_expect_difference()
+{
+ compare_image_with_snapshot "$@" 'true'
+}
+
demote_image()
{
local cluster=$1
local image=$3
local mode=${4:-${RBD_MIRROR_MODE}}
- rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode}
+ run_cmd "rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode}"
# Display image info including the global image id for debugging purpose
- rbd --cluster=${cluster} info ${pool}/${image}
+ run_cmd "rbd --cluster=${cluster} info ${pool}/${image}"
}
test_image_present()
test "${test_state}" = "${current_state}"
}
+test_image_with_global_id_count()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local global_id=$4
+ local test_image_count=$5
+
+ run_cmd "rbd --cluster ${cluster} info ${pool}/${image} --format xml --pretty-format"
+ test "${test_image_count}" = "$($XMLSTARLET sel -t -v "count(//image/mirroring[global_id='${global_id}'])" < "$CMD_STDOUT")" || { fail; return 1; }
+}
+
+test_image_count()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local test_image_count=$4
+
+ run_cmd "rbd --cluster ${cluster} ls ${pool} --format xml --pretty-format"
+ test "${test_image_count}" = "$($XMLSTARLET sel -t -v "count(//images[name='${image}'])" < "$CMD_STDOUT")" || { fail; return 1; }
+}
+
+test_image_with_global_id_not_present()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local global_id=$4
+
+ # if the image is not listed in the pool then no need to check the global id
+ test_image_count "${cluster}" "${pool}" "${image}" 0 && return 0;
+
+ test_image_with_global_id_count "${cluster}" "${pool}" "${image}" "${global_id}" 0 || { fail "image present"; return 1; }
+}
+
+test_image_with_global_id_present()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local global_id=$4
+
+ # if the image is not listed in the pool then no need to check the global id
+ test_image_count "${cluster}" "${pool}" "${image}" 1 || return 1;
+
+ test_image_with_global_id_count "${cluster}" "${pool}" "${image}" "${global_id}" 1
+}
+
+test_image_not_present()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local image_id=$4
+
+ test_image_present "${cluster}" "${pool}" "${image}" 'deleted' "${image_id}"
+}
+
wait_for_image_present()
{
local cluster=$1
sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
}
+get_image_mirroring_global_id()
+{
+ local cluster=$1
+ local image_spec=$2
+ local -n _global_id=$3
+
+ run_cmd "rbd --cluster ${cluster} info ${image_spec} --format xml --pretty-format"
+ _global_id=$($XMLSTARLET sel -t -v "//image/mirroring/global_id" "$CMD_STDOUT") || { fail "not mirrored"; return; }
+}
+
+image_resize()
+{
+ local cluster=$1 ; shift
+ local image_spec=$1 ; shift
+ local size=$1 ; shift
+
+ run_cmd "rbd --cluster ${cluster} resize --image ${image_spec} --size ${size} $*"
+}
+
+get_image_size()
+{
+ local cluster=$1
+ local image_spec=$2
+ local -n _size=$3
+
+ run_cmd "rbd --cluster ${cluster} info ${image_spec} --format xml --pretty-format"
+ _size=$($XMLSTARLET sel -t -v "//image/size" "$CMD_STDOUT") || { fail "unable to determine size"; return; }
+}
+
+test_image_size_matches()
+{
+ local cluster=$1
+ local image_spec=$2
+ local test_size=$3
+
+ local current_size
+ get_image_size "${cluster}" "${image_spec}" current_size
+ test "${current_size}" = "${test_size}" || { fail; return 1; }
+}
+
+wait_for_image_size_matches()
+{
+ local cluster=$1
+ local image_spec=$2
+ local test_size=$3
+ local s
+
+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+ sleep ${s}
+ test_image_size_matches "${cluster}" "${image_spec}" "${test_size}" && return 0
+ done
+ fail "size never matched"; return 1
+}
+
request_resync_image()
{
local cluster=$1
local cluster=$1
local pool=$2
local obj_name=$3
-
- rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name}
+ run_cmd "rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name}"
}
count_omap_keys_with_filter()
local pool=$2
local obj_name=$3
local filter=$4
+ local -n _count=$5
- list_omap_keys ${cluster} ${pool} ${obj_name} | grep -c ${filter}
+ list_omap_keys "${cluster}" "${pool}" "${obj_name}"
+ _count=$(grep -c "${filter}" "$CMD_STDOUT") || return 0
}
wait_for_omap_keys()
local obj_name=$3
local filter=$4
+ local key_count
for s in 0 1 2 2 4 4 8 8 8 16 16 32; do
sleep $s
-
- set +e
- test "$(count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter})" = 0
- error_code=$?
- set -e
-
- if [ $error_code -eq 0 ]; then
- return 0
- fi
+ count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter} key_count
+ test "${key_count}" = 0 && return 0
done
-
+ fail "wait for count of keys 0 failed on ${cluster}. Actual count=${key_count}"
return 1
}
run_cmd "rbd --cluster ${cluster} group remove ${group_spec}"
}
+wait_for_group_synced()
+{
+ local cluster=$1
+ local group_spec=$2
+
+ # Determine secondary cluster
+ local secondary_cluster
+ get_fields_from_mirror_group_status "${cluster}" "${group_spec}" secondary_cluster "(//group/peer_sites/peer_site/site_name)"
+
+ local secondary_group_spec
+ IFS='/' read -r -a group_fields <<< "${group_spec}"
+ if [ "${#group_fields[@]}" -eq 3 ]; then
+ local pool local_namespace secondary_namespace group
+ pool="${group_fields[0]}"
+ local_namespace="${group_fields[1]}"
+ group="${group_fields[2]}"
+
+ # Determine secondary cluster namespace
+ run_cmd "rbd --cluster ${cluster} mirror pool info ${pool}/${local_namespace} --format xml --pretty-format" || { fail; return 1; }
+ secondary_namespace=$(xmlstarlet sel -t -v "//${pool}/remote_namespace" < ${CMD_STDOUT}) || { fail "no remote namespace"; return 1; }
+ secondary_group_spec="${pool}/${secondary_namespace}/${group}"
+ else
+ secondary_group_spec="${group_spec}"
+ fi
+
+ local group_snap_id
+ get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id
+ wait_for_group_snap_present "${secondary_cluster}" "${secondary_group_spec}" "${group_snap_id}"
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${secondary_group_spec}" "${group_snap_id}"
+}
+
group_image_add()
{
local cluster=$1
done
}
+mirror_group_internal()
+{
+ local cmd=$1
+ local group_spec=$2
+ local mode=${3:-${MIRROR_IMAGE_MODE}}
+
+ run_cmd "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}"
+}
+
mirror_group_enable()
{
local cluster=$1
local group_spec=$2
local mode=${3:-${MIRROR_IMAGE_MODE}}
+ local runner=${4:-"run_cmd"}
- run_cmd "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}"
+ "$runner" "rbd --cluster=${cluster} mirror group enable ${group_spec} ${mode}"
+}
+
+mirror_group_enable_try()
+{
+ local mode=${3:-${MIRROR_IMAGE_MODE}}
+ mirror_group_enable "$@" "${mode}" "try_cmd"
}
mirror_group_disable()
local cluster=$1
local group_spec=$2
- run_cmd "rbd --cluster=${cluster} mirror group snapshot ${group_spec}"
+ run_cmd "rbd --cluster=${cluster} mirror group snapshot ${group_spec}" || return 1
+
+ if [ "$#" -gt 2 ]
+ then
+ local -n _group_snap_id=$3
+ _group_snap_id=$(awk -F': ' '{print $NF}' "$CMD_STDOUT" )
+ fi
}
group_snap_create()
local group_spec=$2
local snap=$3
local -n _group_snap_count=$4
+
+ run_cmd "rbd --cluster=${cluster} group snap ls --format xml --pretty-format ${group_spec}"
+ if [ "${snap}" = '*' ]; then
+ _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap)" < "$CMD_STDOUT")"
+ else
+ _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[snapshot='${snap}'])" < "$CMD_STDOUT")"
+ fi
+}
+
+get_group_snap_name()
+{
+ local cluster=$1
+ local group_spec=$2
+ local snap_id=$3
+ local -n _group_snap_name=$4
run_cmd "rbd --cluster=${cluster} group snap ls --format xml --pretty-format ${group_spec}"
- _group_snap_count="$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[snapshot='${snap}'])" < "$CMD_STDOUT")"
+ _group_snap_name="$($XMLSTARLET sel -t -v "//group_snaps/group_snap[id='${snap_id}']/snapshot" < "$CMD_STDOUT")"
+}
+
+get_image_snap_id_from_group_snap_info()
+{
+ local cluster=$1
+ local snap_spec=$2
+ local image_spec=$3
+ local -n _image_snap_id=$4
+
+ run_cmd "rbd --cluster=${cluster} group snap info --format xml --pretty-format ${snap_spec}"
+ local image_name
+ image_name=$(echo "${image_spec}" | awk -F'/' '{print $NF}')
+ _image_snap_id="$($XMLSTARLET sel -t -v "//group_snapshot/images/image[image_name='${image_name}']/snap_id" < "$CMD_STDOUT")"
+}
+
+get_images_from_group_snap_info()
+{
+ local cluster=$1
+ local snap_spec=$2
+ local -n _images=$3
+
+ run_cmd "rbd --cluster=${cluster} group snap info --format xml --pretty-format ${snap_spec}"
+ # sed script removes extra path delimiter if namespace field is blank
+ _images="$($XMLSTARLET sel -t -m "//group_snapshot/images/image" -v "pool_name" -o "/" -v "namespace" -o "/" -v "image_name" -o " " < "$CMD_STDOUT" | sed s/"\/\/"/"\/"/g )"
+}
+
+get_image_snap_complete()
+{
+ local cluster=$1
+ local image_spec=$2
+ local snap_id=$3
+ local -n _is_complete=$4
+ run_cmd "rbd --cluster=${cluster} snap list --all --format xml --pretty-format ${image_spec}"
+ _is_complete="$($XMLSTARLET sel -t -v "//snapshots/snapshot[id='${snap_id}']/namespace/complete" < "$CMD_STDOUT")"
}
check_group_snap_doesnt_exist()
wait_for_test_group_present "${cluster}" "${pool}" "${group}" 0 0
}
+test_group_snap_present()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+ local expected_snap_count=$4
+
+ # TODO - have seen this next cmd fail with rc=2 and an empty list
+ # this should not happen, but if it does then retry as a temp workaround
+ try_cmd "rbd --cluster ${cluster} group snap list ${group_spec} --format xml --pretty-format"
+
+ test "${expected_snap_count}" = "$($XMLSTARLET sel -t -v "count(//group_snaps/group_snap[id='${group_snap_id}'])" < "$CMD_STDOUT")" || { fail; return 1; }
+}
+
+wait_for_test_group_snap_present()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+ local test_group_snap_count=$4
+ local s
+
+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+ sleep ${s}
+ test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" "${test_group_snap_count}" && return 0
+ done
+
+ fail "wait for count of group snaps with id ${group_snap_id} to be ${test_group_snap_count} failed on ${cluster}"
+ return 1
+}
+
+wait_for_group_snap_present()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+
+ wait_for_test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" 1
+}
+
+wait_for_group_snap_not_present()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+
+ wait_for_test_group_snap_present "${cluster}" "${group_spec}" "${group_snap_id}" 0
+}
+
+test_group_snap_sync_state()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+ local expected_state=$4
+
+ # TODO - have seen this next cmd fail with rc=2 and an empty list
+ # this should not happen, but if it does then retry as a temp workaround
+ try_cmd "rbd --cluster ${cluster} group snap list ${group_spec} --format xml --pretty-format"
+
+ test "${expected_state}" = "$($XMLSTARLET sel -t -v "//group_snaps/group_snap[id='${group_snap_id}']/state" < "$CMD_STDOUT")" || { fail; return 1; }
+}
+
+test_group_snap_sync_complete()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+
+ test_group_snap_sync_state "${cluster}" "${group_spec}" "${group_snap_id}" 'complete'
+}
+
+test_group_snap_sync_incomplete()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+
+ test_group_snap_sync_state "${cluster}" "${group_spec}" "${group_snap_id}" 'incomplete'
+}
+
+wait_for_test_group_snap_sync_complete()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+ local s
+
+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+ sleep ${s}
+ test_group_snap_sync_complete "${cluster}" "${group_spec}" "${group_snap_id}" && return 0
+ done
+
+ fail "wait for group snap with id ${group_snap_id} to be synced failed on ${cluster}"
+ return 1
+}
+
+wait_for_group_snap_sync_complete()
+{
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+
+ wait_for_test_group_snap_sync_complete "${cluster}" "${group_spec}" "${group_snap_id}"
+}
+
test_group_replay_state()
{
local cluster=$1
fi
}
+query_replayer_assignment()
+{
+ local cluster=$1
+ local instance=$2
+ local -n _result=$3
+
+ local group_replayers
+ local image_replayers
+ local group_replayers_count
+ local image_replayers_count
+
+ admin_daemon "${cluster}:${instance}" rbd mirror status --format xml-pretty || { fail; return 1; }
+ group_replayers=$($XMLSTARLET sel -t -v "//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/name" < "$CMD_STDOUT") || { group_replayers=''; }
+ image_replayers=$($XMLSTARLET sel -t -v "//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/image_replayers/image_replayer/name" < "$CMD_STDOUT") || { image_replayers=''; }
+ group_replayers_count=$($XMLSTARLET sel -t -v "count(//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/name)" < "$CMD_STDOUT") || { group_replayers_count='0'; }
+ image_replayers_count=$($XMLSTARLET sel -t -v "count(//mirror_status/pool_replayers/pool_replayer_status/group_replayers/group_replayer/image_replayers/image_replayer/name)" < "$CMD_STDOUT") || { image_replayers_count='0'; }
+ _result=("${group_replayers}" "${image_replayers}" "${group_replayers_count}" "${image_replayers_count}")
+}
+
wait_for_group_replay_state()
{
local cluster=$1
mirror_group_snapshot_and_wait_for_sync_complete()
{
- local local_cluster=$1
- local cluster=$2
+ local secondary_cluster=$1
+ local primary_cluster=$2
local group_spec=$3
local group_snap_id
- local local_group_snap_id
if [ "${MIRROR_IMAGE_MODE}" != "snapshot" ]; then
return 1
fi
- mirror_group_snapshot "${cluster}" "${group_spec}"
- get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id
-
- while true; do
- for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
- sleep ${s}
-
- get_newest_group_mirror_snapshot_id "${local_cluster}" "${group_spec}" local_group_snap_id
- test "${local_group_snap_id}" = "${group_snap_id}" && return 0
- done
- fail "Failed to reach expected state"
- return 1
- done
- return 1
+ mirror_group_snapshot "${primary_cluster}" "${group_spec}" group_snap_id
+ wait_for_group_snap_present "${secondary_cluster}" "${group_spec}" "${group_snap_id}"
+ wait_for_group_snap_sync_complete "${secondary_cluster}" "${group_spec}" "${group_snap_id}"
}
-test_group_and_image_sync_status()
+test_group_synced_image_status()
{
- local local_cluster=$1
- local cluster=$2
- local group_spec=$3
- local image_spec=$4
+ local cluster=$1
+ local group_spec=$2
+ local group_snap_id=$3
+ local expected_synced_image_count=$4
- local group_snap_id
- local local_group_snap_id
+ local group_snap_name
+ get_group_snap_name "${cluster}" "${group_spec}" "${group_snap_id}" group_snap_name
- get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id
+ local images
+ get_images_from_group_snap_info "${cluster}" "${group_spec}@${group_snap_name}" images
- local local_image_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${group_spec}-${image_spec}-${local_group_snap_id}.status)
+ local image_count=0
+ local image_spec
+ for image_spec in ${images}; do
- while true; do
- for s in 0.4 1 1 1 2 2 2 4 8 16 16 32 32; do
- sleep ${s}
+ # get the snap_id for this image from the group snap info
+ local image_snap_id
+ get_image_snap_id_from_group_snap_info "${cluster}" "${group_spec}@${group_snap_name}" "${image_spec}" image_snap_id
+
+ # get the value in the "complete" field for the image and snap_id
+ local is_complete
+ get_image_snap_complete "${cluster}" "${image_spec}" "${image_snap_id}" is_complete
- get_newest_group_mirror_snapshot_id "${local_cluster}" "${group_spec}" local_group_snap_id
- if [ "${local_group_snap_id}" == "${group_snap_id}" ]; then
-#TODO: Get the image snap for this group snap. For now, compare against the last mirror image .group snap.
-# use rbd group snap info <snap_name> for this once it is working properly
- rbd --cluster "${local_cluster}" snap list --all "${image_spec}" --format xml | \
- $XMLSTARLET sel -t -c "//snapshots/snapshot[namespace/type='mirror' and position()=last()]" > ${local_image_status_log}
- local image_snap_complete=$(xmlstarlet sel -t -v "//snapshot/namespace/complete" < ${local_image_status_log})
- if [ "${image_snap_complete}" != "true" ]; then
- return 1
- else
- return 0
- fi
- else
- continue
- fi
- done
- return 1
+ test "${is_complete}" != "true" && { fail "image ${image_spec} is not synced"; return 1; }
+
+ image_count=$((image_count+1))
done
+
+ test "${image_count}" != "${expected_synced_image_count}" && fail "unexpected count ${image_count} != ${expected_synced_image_count}"
+
+ return 0
+}
+
+test_images_in_latest_synced_group()
+{
+ local cluster=$1
+ local group_spec=$2
+ local expected_synced_image_count=$3
+
+ local group_snap_id
+ get_newest_group_mirror_snapshot_id "${cluster}" "${group_spec}" group_snap_id
+ test_group_synced_image_status "${cluster}" "${group_spec}" "${group_snap_id}" "${expected_synced_image_count}"
}
test_group_status_in_pool_dir()
test "${image_count}" = "${actual_image_count}" || { fail; return 1; }
# If the group is started then check that all images are started too
- test "${current_state}" = "started" || return 0
- test "${image_count}" = "${started_image_count}" || { fail; return 1; }
+ if [ "${current_state}" = "started" ]; then
+ test "${image_count}" = "${started_image_count}" || { fail; return 1; }
+ fi
fi
+ # TODO enable this once tests are more reliable
+ #check_fields_in_group_and_image_status "${cluster}" "${group_spec}" || { fail; return 1; }
+
return 0
}
stop_mirrors ${cluster} '-9'
done
+ for cluster in ${primary_cluster} ${secondary_cluster}; do
+ echo 'cluster:'${cluster}
+ for pool in $(CEPH_ARGS='' ceph --cluster ${cluster} osd pool ls | grep -v "^\." | xargs); do
+ echo 'pool:'${pool}
+ run_admin_cmd "ceph --cluster ${cluster} osd pool delete ${pool} ${pool} --yes-i-really-really-mean-it"
+ done
+ done
+
+ # following is old method that used to remove individual object rather than removing entire pools
+ : '
for cluster in ${primary_cluster} ${secondary_cluster}; do
echo 'cluster:'${cluster}
for pool in "${POOL}" "${PARENT_POOL}" "${POOL}/${NS1}" "${POOL}/${NS2}"; do
done
done
done
-
+ '
}
# list all groups, images and snaps
projects / ceph.git / commitdiff