From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 1 Sep 2023 12:53:33 +0000 (+0200)
Subject: qa: add "failover / failback loop" test for rbd-mirror
X-Git-Tag: v19.0.0~567^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=153df2d64b8cb51b5ce7559576613788795a4967;p=ceph-ci.git

qa: add "failover / failback loop" test for rbd-mirror

For snapshot-based mirroring, check that demote (or other mirror
snapshots) don't pile up.  Nothing in particular to assert on for
journal-based mirroring but the test is still useful.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index ca715d854c1..f4961b925e6 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -1169,6 +1169,16 @@ wait_for_snap_removed_from_trash()
     return 1
 }
 
+count_mirror_snaps()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} snap ls ${pool}/${image} --all |
+        grep -c -F " mirror ("
+}
+
 write_image()
 {
     local cluster=$1
diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh
index 56a8b13a9a7..54f6aeec8e0 100755
--- a/qa/workunits/rbd/rbd_mirror_journal.sh
+++ b/qa/workunits/rbd/rbd_mirror_journal.sh
@@ -214,7 +214,29 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
-# force promote
+testlog "TEST: failover / failback loop"
+for i in `seq 1 20`; do
+  demote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
+  demote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+done
+
+testlog "TEST: force promote"
 force_promote_image=test_force_promote
 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh
index 0060440fb8d..c70d48b09db 100755
--- a/qa/workunits/rbd/rbd_mirror_snapshot.sh
+++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh
@@ -220,7 +220,32 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
-# force promote
+testlog "TEST: failover / failback loop"
+for i in `seq 1 20`; do
+  demote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
+  demote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+done
+# check that demote (or other mirror snapshots) don't pile up
+test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3
+test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3
+
+testlog "TEST: force promote"
 force_promote_image=test_force_promote
 create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image}
 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100