From f9b019f53effbec4294474418cf4c0473fe18751 Mon Sep 17 00:00:00 2001
From: Mykola Golub <mgolub@suse.com>
Date: Fri, 12 Mar 2021 11:57:57 +0000
Subject: [PATCH] qa/workunits/rbd: mirror group functional tests

Also sets the RBD_MIRROR_INSTANCES to 1 to avoid any deviations.

Signed-off-by: Mykola Golub <mgolub@suse.com>
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---
 qa/workunits/rbd/rbd_mirror_group.sh   | 267 ++++++++++++++++++++
 qa/workunits/rbd/rbd_mirror_helpers.sh | 328 ++++++++++++++++++++++++-
 2 files changed, 592 insertions(+), 3 deletions(-)
 create mode 100755 qa/workunits/rbd/rbd_mirror_group.sh

diff --git a/qa/workunits/rbd/rbd_mirror_group.sh b/qa/workunits/rbd/rbd_mirror_group.sh
new file mode 100755
index 0000000000000..15f3a6de1ffce
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_group.sh
@@ -0,0 +1,267 @@
+#!/usr/bin/env bash
+#
+# rbd_mirror_group.sh - test rbd-mirror daemon for snapshot-based group mirroring
+#
+# The script starts two ("local" and "remote") clusters using mstart.sh script,
+# creates a temporary directory, used for cluster configs, daemon logs, admin
+# socket, temporary files, and launches rbd-mirror daemon.
+#
+
+set -ex
+
+MIRROR_POOL_MODE=image
+MIRROR_IMAGE_MODE=snapshot
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
+testlog "TEST: create group and test replay"
+start_mirrors ${CLUSTER1}
+group=test-group
+create_group_and_enable_mirror ${CLUSTER2} ${POOL} ${group}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 0
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'present'
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group} 'down+unknown'
+fi
+
+testlog "TEST: add image to group and test replay"
+image=test-image
+create_image ${CLUSTER2} ${POOL} ${image}
+group_add_image ${CLUSTER2} ${POOL} ${group} ${POOL} ${image}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
+
+testlog "TEST: test replay with remove image and later add the same"
+group_remove_image ${CLUSTER2} ${POOL} ${group} ${POOL} ${image}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 0
+group_add_image ${CLUSTER2} ${POOL} ${group} ${POOL} ${image}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+
+testlog "TEST: stop mirror, create group, start mirror and test replay"
+stop_mirrors ${CLUSTER1}
+group1=test-group1
+create_group_and_enable_mirror ${CLUSTER2} ${POOL} ${group1}
+image1=test-image1
+create_image ${CLUSTER2} ${POOL} ${image1}
+group_add_image ${CLUSTER2} ${POOL} ${group1} ${POOL} ${image1}
+start_mirrors ${CLUSTER1}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group1} 1
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+replaying'
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'down+unknown'
+fi
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
+
+testlog "TEST: test the first group is replaying after restart"
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
+
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  testlog "TEST: stop/start/restart group via admin socket"
+
+  admin_daemons ${CLUSTER1} rbd mirror group stop ${POOL}/${group1}
+  wait_for_group_replay_stopped ${CLUSTER1} ${POOL} ${group1} 1
+  wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+stopped'
+
+  admin_daemons ${CLUSTER1} rbd mirror group start ${POOL}/${group1}
+  wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group1} 1
+  wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+replaying'
+
+  admin_daemons ${CLUSTER1} rbd mirror group restart ${POOL}/${group1}
+  wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group1} 1
+  wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+replaying'
+
+  flush ${CLUSTER1}
+  admin_daemons ${CLUSTER1} rbd mirror group status ${POOL}/${group1}
+fi
+
+testlog "TEST: test group rename"
+new_name="${group}_RENAMED"
+rename_group ${CLUSTER2} ${POOL} ${group} ${new_name}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${new_name} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
+rename_group ${CLUSTER2} ${POOL} ${new_name} ${group}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+
+testlog "TEST: failover and failback"
+start_mirrors ${CLUSTER2}
+
+testlog " - demote and promote same cluster"
+demote_group ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL} ${group1} 1
+
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'up+stopped'
+promote_group ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group1} 1
+
+write_image ${CLUSTER2} ${POOL} ${image1} 100
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group1}
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
+
+testlog " - failover (unmodified)"
+demote_group ${CLUSTER2} ${POOL} ${group}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group} 'up+stopped'
+promote_group ${CLUSTER1} ${POOL} ${group}
+wait_for_group_replay_started ${CLUSTER2} ${POOL} ${group} 1
+
+testlog " - failback (unmodified)"
+demote_group ${CLUSTER1} ${POOL} ${group}
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group} 'up+stopped'
+promote_group ${CLUSTER2} ${POOL} ${group}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group} 'up+stopped'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
+
+testlog " - failover"
+demote_group ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL} ${group1} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'up+stopped'
+promote_group ${CLUSTER1} ${POOL} ${group1}
+wait_for_group_replay_started ${CLUSTER2} ${POOL} ${group1} 1
+write_image ${CLUSTER1} ${POOL} ${image1} 100
+wait_for_group_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${group1}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
+
+testlog " - failback"
+demote_group ${CLUSTER1} ${POOL} ${group1}
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL} ${group1} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'up+stopped'
+promote_group ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group1} 1
+write_image ${CLUSTER2} ${POOL} ${image1} 100
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group1}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group1} 'up+replaying'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group1} 'up+stopped'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
+
+testlog " - force promote"
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${group}
+promote_group ${CLUSTER1} ${POOL} ${group} '--force'
+
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+stopped'
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL} ${group} 'up+stopped'
+write_image ${CLUSTER1} ${POOL} ${image} 100
+write_image ${CLUSTER2} ${POOL} ${image} 100
+group_remove_image ${CLUSTER1} ${POOL} ${group} ${POOL} ${image}
+remove_image_retry ${CLUSTER1} ${POOL} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+remove_group ${CLUSTER1} ${POOL} ${group}
+
+testlog "TEST: disable mirroring / delete non-primary group"
+disable_group_mirror ${CLUSTER2} ${POOL} ${group}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'deleted'
+enable_group_mirror ${CLUSTER2} ${POOL} ${group}
+wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'present' 1
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+
+testlog "TEST: disable mirror while daemon is stopped"
+stop_mirrors ${CLUSTER1}
+stop_mirrors ${CLUSTER2}
+disable_group_mirror ${CLUSTER2} ${POOL} ${group}
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'present' 1
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+fi
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'deleted'
+enable_group_mirror ${CLUSTER2} ${POOL} ${group}
+wait_for_group_present ${CLUSTER1} ${POOL} ${group} 'present' 1
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+
+testlog "TEST: non-default namespace group mirroring"
+testlog " - replay"
+create_group_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${group}
+create_group_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${group}
+create_image ${CLUSTER2} ${POOL}/${NS1} ${image}
+create_image ${CLUSTER2} ${POOL}/${NS2} ${image}
+group_add_image ${CLUSTER2} ${POOL}/${NS1} ${group} ${POOL}/${NS1} ${image}
+group_add_image ${CLUSTER2} ${POOL}/${NS2} ${group} ${POOL}/${NS2} ${image}
+wait_for_group_replay_started ${CLUSTER1} ${POOL}/${NS1} ${group} 1
+wait_for_group_replay_started ${CLUSTER1} ${POOL}/${NS2} ${group} 1
+write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
+write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${group}
+wait_for_group_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${group} 'up+replaying'
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${group} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${image}
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${POOL}/${NS2} ${image}
+
+testlog " - disable mirroring / remove group"
+group_remove_image ${CLUSTER2} ${POOL}/${NS1} ${group} ${POOL}/${NS1} ${image}
+remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted'
+remove_group ${CLUSTER1} ${POOL}/${NS1} ${group}
+wait_for_group_present ${CLUSTER1} ${POOL}/${NS1} ${group} 'deleted'
+disable_group_mirror ${CLUSTER2} ${POOL}/${NS2} ${group}
+wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted'
+wait_for_group_present ${CLUSTER1} ${POOL}/${NS2} ${group} 'deleted'
+
+testlog "TEST: simple group resync"
+image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
+wait_for_image_present ${CLUSTER2} ${POOL} ${image} 'present'
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+request_resync_group ${CLUSTER1} ${POOL} ${group}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
+
+testlog "TEST: request group resync while daemon is offline"
+image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
+stop_mirrors ${CLUSTER1}
+request_resync_group ${CLUSTER1} ${POOL} ${group}
+start_mirrors ${CLUSTER1}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_group_replay_started ${CLUSTER1} ${POOL} ${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
+
+testlog "TEST: split-brain"
+promote_group ${CLUSTER1} ${POOL} ${group} --force
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+stopped'
+write_image ${CLUSTER1} ${POOL} ${image} 10
+demote_group ${CLUSTER1} ${POOL} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+error' 'split-brain'
+request_resync_group ${CLUSTER1} ${POOL} ${group}
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL} ${group} 'up+replaying'
+
+#TODO: Fix blocklisting IP's which are consequence of "TEST: stop mirror, create group, start mirror and test replay"
+#if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+# teuthology will trash the daemon
+#  testlog "TEST: no blocklists"
+#  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
+#  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
+#fi
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index b94317d75d3a5..dd0db96ef40c3 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -72,7 +72,16 @@
 #     ../qa/workunits/rbd/rbd_mirror_helpers.sh cleanup
 #
 
-RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2}
+if type xmlstarlet > /dev/null 2>&1; then
+    XMLSTARLET=xmlstarlet
+elif type xml > /dev/null 2>&1; then
+    XMLSTARLET=xml
+else
+    echo "Missing xmlstarlet binary!"
+    exit 1
+fi
+
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-1}
 
 CLUSTER1=cluster1
 CLUSTER2=cluster2
@@ -214,7 +223,6 @@ setup_cluster()
     local cluster=$1
 
     CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS}
-
     cd ${CEPH_ROOT}
     rm -f ${TEMPDIR}/${cluster}.conf
     ln -s $(readlink -f run/${cluster}/ceph.conf) \
@@ -444,8 +452,8 @@ stop_mirror()
     pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || :
     if [ -n "${pid}" ]
     then
-        kill ${sig} ${pid}
         for s in 1 2 4 8 16 32; do
+            kill ${sig} ${pid}
             sleep $s
             ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break
         done
@@ -1556,6 +1564,320 @@ wait_for_image_in_omap()
     wait_for_omap_keys ${cluster} ${pool} rbd_mirror_leader image_map
 }
 
+create_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster ${cluster} group create ${pool}/${group}
+}
+
+rename_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local new_name=$4
+
+    rbd --cluster ${cluster} group rename ${pool}/${group} ${pool}/${new_name}
+}
+
+remove_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster ${cluster} group remove ${pool}/${group}
+}
+
+group_add_image()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local image_pool=$4
+    local image=$5
+
+    rbd --cluster ${cluster} \
+        group image add ${pool}/${group} ${image_pool}/${image}
+}
+
+group_remove_image()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local image_pool=$4
+    local image=$5
+
+    rbd --cluster ${cluster} \
+        group image remove ${pool}/${group} ${image_pool}/${image}
+}
+
+enable_group_mirror()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local mode=${4:-${MIRROR_IMAGE_MODE}}
+
+    rbd --cluster=${cluster} mirror group enable ${pool}/${group} ${mode}
+}
+
+disable_group_mirror()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster=${cluster} mirror group disable ${pool}/${group}
+}
+
+create_group_and_enable_mirror()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local mode=${4:-${MIRROR_IMAGE_MODE}}
+
+    create_group ${cluster} ${pool} ${group}
+    enable_group_mirror ${cluster} ${pool} ${group} ${mode}
+}
+
+demote_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster=${cluster} mirror group demote ${pool}/${group}
+}
+
+promote_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local force=$4
+
+    rbd --cluster=${cluster} mirror group promote ${pool}/${group} ${force}
+}
+
+mirror_group_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster=${cluster} mirror group snapshot ${pool}/${group}
+}
+
+request_resync_group()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+
+    rbd --cluster=${cluster} mirror group resync ${pool}/${group}
+}
+
+test_group_present()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local test_state=$4
+    local test_image_count=$5
+    local current_state=deleted
+    local current_image_count
+
+    rbd --cluster ${cluster} group list ${pool} | grep "^${group}$" &&
+        current_state=present
+    test "${test_state}" = "${current_state}" || return 1
+
+    test "${current_state}" = present -a -n "${image_count}" || return 0
+
+    current_image_count=$(rbd --cluster ${cluster} group image list ${pool}/${group} | wc -l)
+    test "${test_image_count}" = "${current_image_count}"
+}
+
+wait_for_group_present()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local state=$4
+    local image_count=$5
+    local s
+
+    for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+        sleep ${s}
+        test_group_present \
+            "${cluster}" "${pool}" "${group}" "${state}" "${image_count}" &&
+        return 0
+    done
+    return 1
+}
+
+test_group_replay_state()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local test_state=$4
+    local image_count=$5
+    local status_result
+    local current_state=stopped
+
+    status_result=$(admin_daemons "${cluster}" rbd mirror group status ${pool}/${group} | grep -i 'state') || return 1
+    echo "${status_result}" | grep -i 'Replaying' && current_state=started
+    test "${test_state}" = "${current_state}" || return 1
+
+    test -n "${image_count}" || return 0
+
+    # TODO: test images when group image status is returned by the rbd-mirror
+}
+
+wait_for_group_replay_state()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local state=$4
+    local image_count=$5
+    local s
+
+    # TODO: add a way to force rbd-mirror to update replayers
+    for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+        sleep ${s}
+        test_group_replay_state "${cluster}" "${pool}" "${group}" "${state}" \
+                                "${image_count}" &&
+            return 0
+    done
+    return 1
+}
+
+wait_for_group_replay_started()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local image_count=$4
+
+    wait_for_group_replay_state "${cluster}" "${pool}" "${group}" started "${image_count}"
+}
+
+wait_for_group_replay_stopped()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local image_count=$4
+
+    wait_for_group_replay_state "${cluster}" "${pool}" "${group}" stopped "${image_count}"
+}
+
+get_newest_group_mirror_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local log=$4
+
+    rbd --cluster "${cluster}" group snap list "${pool}/${group}" --format xml | \
+        xmlstarlet sel -t -c "//group_snaps/group_snap[state='complete' and position()=last()]" > \
+        ${log} || true
+}
+
+wait_for_group_snapshot_sync_complete()
+{
+    local local_cluster=$1
+    local cluster=$2
+    local pool=$3
+    local group=$4
+
+    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${group}.status)
+    local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${group}.status)
+
+    mirror_group_snapshot "${cluster}" "${pool}" "${group}"
+    get_newest_group_mirror_snapshot "${cluster}" "${pool}" "${group}" "${status_log}"
+    local group_snap_id=$(xmlstarlet sel -t -v "//group_snap/snapshot" < ${status_log} | awk -F. '{print $NF}')
+
+    while true; do
+        for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
+            sleep ${s}
+
+            get_newest_group_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}"
+            local local_group_snap_id=$(xmlstarlet sel -t -v "//group_snap/snapshot" < ${local_status_log} | awk -F. '{print $NF}')
+
+            test "${local_group_snap_id}" = "${group_snap_id}" && return 0
+        done
+
+        return 1
+    done
+    return 1
+}
+
+wait_for_group_replay_complete()
+{
+    local local_cluster=$1
+    local cluster=$2
+    local pool=$3
+    local group=$4
+
+    if [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then
+        wait_for_group_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${group}
+    else
+        return 1
+    fi
+}
+
+test_group_status_in_pool_dir()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
+
+    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${group}.mirror_status)
+    CEPH_ARGS='' rbd --cluster ${cluster} mirror group status ${pool}/${group} |
+        tee ${status_log} >&2
+    grep "^  state: .*${state_pattern}" ${status_log} || return 1
+    grep "^  description: .*${description_pattern}" ${status_log} || return 1
+
+    if [ -n "${service_pattern}" ]; then
+        grep "service: *${service_pattern}" ${status_log} || return 1
+    elif echo ${state_pattern} | grep '^up+'; then
+        grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1
+    else
+        grep "service: " ${status_log} && return 1
+    fi
+
+    return 0
+}
+
+wait_for_group_status_in_pool_dir()
+{
+    local cluster=$1
+    local pool=$2
+    local group=$3
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
+
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+        sleep ${s}
+        test_group_status_in_pool_dir ${cluster} ${pool} ${group} \
+            "${state_pattern}" "${description_pattern}" "${service_pattern}" &&
+            return 0
+    done
+    return 1
+}
+
 #
 # Main
 #
-- 
2.39.5