qa/workunits/rbd: test snapshot sync with interrupted daemon on secondary

Validate the snapshot mirroring functionality when the rbd-mirror daemon on
the secondary cluster is interrupted during an snapshot sync process.

Signed-off-by: Miki Patel <miki.patel132@gmail.com>

diff --git a/qa/workunits/rbd/rbd_mirror_group_simple.sh b/qa/workunits/rbd/rbd_mirror_group_simple.sh
index 7a2fdcc40ee07e1ea6a49fd6b694a39b06a05647..234c3bf19dc3abf88968c1d1cee5af77f0ee910b 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_group_simple.sh
+++ b/qa/workunits/rbd/rbd_mirror_group_simple.sh
@@ -2931,6 +2931,66 @@ test_force_promote_before_initial_sync()
   start_mirrors "${secondary_cluster}"
 }
 
+# test snapshot sync when rbd-mirror daemon on secondary cluster is interrupted
+declare -a test_interrupted_sync_restarted_daemon_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}" 3 2048)
+
+test_interrupted_sync_restarted_daemon_scenarios=1
+
+test_interrupted_sync_restarted_daemon()
+{
+  local primary_cluster=$1 ; shift
+  local secondary_cluster=$1 ; shift
+  local pool=$1 ; shift
+  local image_count=$1 ; shift
+  local image_size=$1 ; shift
+  local io_count=256
+  local io_size=4096
+  local group_snap_id
+
+  local group0=test-group0
+  local image_prefix="test_image"
+
+  start_mirrors "${primary_cluster}"
+  start_mirrors "${secondary_cluster}"
+
+  group_create "${primary_cluster}" "${pool}/${group0}"
+  images_create "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}" "${image_size}"
+  group_images_add "${primary_cluster}" "${pool}/${group0}" "${pool}/${image_prefix}" "${image_count}"
+
+  mirror_group_enable "${primary_cluster}" "${pool}/${group0}"
+
+  wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
+  wait_for_group_replay_started "${secondary_cluster}" "${pool}"/"${group0}" "${image_count}"
+  wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
+
+  wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}" "${secondary_cluster}" "${pool}"/"${group0}"
+
+  for i in $(seq 0 $(("${image_count}"-1))); do
+    write_image "${primary_cluster}" "${pool}" "${image_prefix}$i" "${io_count}" "${io_size}"
+  done;
+
+  mirror_group_snapshot "${primary_cluster}" "${pool}/${group0}" group_snap_id
+
+  # make sure snapshot is present on secondary cluster
+  wait_for_group_snap_present "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}"
+
+  stop_mirrors "${secondary_cluster}" '-9'
+  # make sure the snapshot remains in an incomplete state after the daemon is stopped
+  test_group_snap_sync_incomplete "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}"
+  start_mirrors "${secondary_cluster}"
+
+  # Finally, after restarting the daemon, make sure the snapshot sync has resumed and progressed to completion
+  wait_for_group_snap_sync_complete "${secondary_cluster}" "${pool}/${group0}" "${group_snap_id}"
+
+  # cleanup
+  mirror_group_disable "${primary_cluster}" "${pool}/${group0}"
+  group_remove "${primary_cluster}" "${pool}/${group0}"
+  wait_for_group_not_present "${primary_cluster}" "${pool}" "${group0}"
+  wait_for_group_not_present "${secondary_cluster}" "${pool}" "${group0}"
+  images_remove "${primary_cluster}" "${pool}/${image_prefix}" "${image_count}"
+}
+
+
 # test force unlink time
 declare -a test_multiple_mirror_group_snapshot_unlink_time_1=("${CLUSTER2}" "${CLUSTER1}" "${pool0}")
 
@@ -3689,6 +3749,7 @@ run_all_tests()
   run_test_all_scenarios test_empty_group_omap_keys
   # TODO: add the capabilty to have clone images support in the mirror group
   run_test_all_scenarios test_group_with_clone_image
+  run_test_all_scenarios test_interrupted_sync_restarted_daemon
   run_test_all_scenarios test_multiple_mirror_group_snapshot_unlink_time
   run_test_all_scenarios test_force_promote_delete_group
   run_test_all_scenarios test_create_group_stop_daemon_then_recreate

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 231be35d4e3081d092e742377620007d3d64b596..8a1981d6e41d3ce2a18979f66955618236c3eb30 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -2394,7 +2394,7 @@ mirror_group_snapshot()
     if [ "$#" -gt 2 ]
     then
       local -n _group_snap_id=$3
-      _group_snap_id=$(awk -F': ' '{print $NF}' "$CMD_STDOUT" )
+      _group_snap_id=$(awk -F": " '/Snapshot ID:/ {print $2}' "$CMD_STDOUT")
     fi
 }