rbd-mirror: allow incomplete demote snapshot to sync after rbd-mirror daemon restart

Currently when the rbd-mirror daemon on secondary was killed while a demote snapshot was newly created (0% copied)
or partially synced, the image's promotion state was set to **PROMOTION_STATE_ORPHAN** upon restart of rbd-mirror
daemon on secondary. This state prevents the demote snapshot sync after restart as bootstrap on secondary fails.
In this commit we fix this by assigning promotion state to **PROMOTION_STATE_NON_PRIMARY**
for an image with an **incomplete non-primary demote snapshot**.

The downside is that if the image is removed on the primary cluster, then after restart of
rbd-mirror daemon on secondary cluster, the corresponding image on the secondary also gets removed.
This is because deletion propagation is unconditionally enabled precisely for **PROMOTION_STATE_NON_PRIMARY**
images and this is okay since the user would have deleted the primary demoted image forcefully.

Fixes: https://tracker.ceph.com/issues/73528
Signed-off-by: VinayBhaskar-V <vvarada@redhat.com>
(cherry picked from commit 636a3929c1a4052827660f44f35842ac62f6d69a)

diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
index 61cc0d6c8f36abf9d85c237bcd76b68c64685ed8..43deb7baf33335584b96787e4f2e339c3e96db94 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror.sh
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -526,6 +526,31 @@ for i in ${image2} ${image4}; do
   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 done
 
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  testlog "TEST: demote image while daemon is offline"
+  demote_image=test_demote_image
+  create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${demote_image} ${RBD_MIRROR_MODE}
+  write_image ${CLUSTER2} ${POOL} ${demote_image} 100
+  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${demote_image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${demote_image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+stopped'
+  stop_mirrors ${CLUSTER1}
+  write_image ${CLUSTER2} ${POOL} ${demote_image} 100
+  demote_image ${CLUSTER2} ${POOL} ${demote_image}
+  start_mirrors ${CLUSTER1}
+  wait_for_snapshot_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+unknown'
+  compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+  promote_image ${CLUSTER1} ${POOL} ${demote_image}
+  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${demote_image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+replaying'
+  remove_image_retry ${CLUSTER1} ${POOL} ${demote_image}
+fi
+
 if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
   testlog "TEST: request image resync when remote is not primary"
   test_resync_image=test_resync_image
@@ -798,6 +823,38 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
 fi
 
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+    testlog "TEST: test partially synced demote snapshot sync after daemon restart"
+    demote_image=test_demote_image
+    create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${demote_image} ${RBD_MIRROR_MODE} 10G
+    write_image ${CLUSTER2} ${POOL} ${demote_image} 100
+    wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${demote_image}
+    wait_for_image_replay_started ${CLUSTER1} ${POOL} ${demote_image}
+    wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+    wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+replaying'
+    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+stopped'
+    write_image ${CLUSTER2} ${POOL} ${demote_image} 2560 4194304
+    demote_image ${CLUSTER2} ${POOL} ${demote_image}
+    get_newest_complete_mirror_snapshot_id ${CLUSTER2} ${POOL} ${demote_image} primary_snap_id
+    wait_for_non_primary_snap_present ${CLUSTER1} ${POOL} ${demote_image} ${primary_snap_id}
+    sleep $((RANDOM % 6))
+    stop_mirrors ${CLUSTER1} -KILL
+    SNAPS=$(get_snaps_json ${CLUSTER1} ${POOL} ${demote_image})
+    jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' <<< ${SNAPS}
+    start_mirrors ${CLUSTER1}
+    wait_for_snapshot_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+unknown'
+    wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+unknown'
+    compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image}
+    promote_image ${CLUSTER1} ${POOL} ${demote_image}
+    wait_for_image_replay_started ${CLUSTER2} ${POOL} ${demote_image}
+    wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+stopped'
+    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+replaying'
+    remove_image_retry ${CLUSTER1} ${POOL} ${demote_image}
+  fi
+fi
+
 testlog "TEST: force promote with a user snapshot"
 force_promote_image=test_force_promote_user
 create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} ${RBD_MIRROR_MODE} 10G

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index e9d4524f58f068011681c0cd8390d37d37911dbc..6dbd950d460ac26447474a28d0587d7112a0e22e 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -795,6 +795,34 @@ get_newest_mirror_snapshot()
         ${log} || true
 }
 
+get_newest_complete_mirror_snapshot_id()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local -n _snap_id=$4
+
+    _snap_id=$(rbd --cluster ${cluster} snap list --all ${pool}/${image} --format xml | \
+        xmlstarlet sel -t -v "(//snapshots/snapshot[namespace/complete='true']/id)[last()]")
+}
+
+wait_for_non_primary_snap_present()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local primary_snap_id=$4
+    local s
+
+    for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+        sleep ${s}
+        rbd --cluster ${cluster} snap list --all ${pool}/${image} --format xml | \
+            xmlstarlet sel -t -c "//snapshots/snapshot/namespace[primary_snap_id='${primary_snap_id}']" && return 0
+    done
+
+    return 1
+}
+
 wait_for_snapshot_sync_complete()
 {
     local local_cluster=$1
@@ -842,7 +870,6 @@ wait_for_replay_complete()
     fi
 }
 
-
 test_status_in_pool_dir()
 {
     local cluster=$1

diff --git a/src/librbd/mirror/GetInfoRequest.cc b/src/librbd/mirror/GetInfoRequest.cc
index e904a21bba1c09e61157b8984b391db113306806..40fbfbff9d71df400f58b574936230272b410bbf 100644 (file)
--- a/src/librbd/mirror/GetInfoRequest.cc
+++ b/src/librbd/mirror/GetInfoRequest.cc
@@ -274,9 +274,15 @@ void GetInfoRequest<I>::calc_promotion_state(
         *m_primary_mirror_uuid = mirror_ns->primary_mirror_uuid;
         break;
       case cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED:
-      case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED:
         *m_promotion_state = PROMOTION_STATE_ORPHAN;
         break;
+      case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED:
+        if (mirror_ns->complete) {
+          *m_promotion_state = PROMOTION_STATE_ORPHAN;
+        } else {
+          *m_promotion_state = PROMOTION_STATE_NON_PRIMARY;
+        }
+        break;
       }
       break;
     }