From d8f73e67894cb0dabb28d4258b32536f77f2c0a9 Mon Sep 17 00:00:00 2001 From: VinayBhaskar-V Date: Tue, 14 Oct 2025 17:17:37 +0530 Subject: [PATCH] rbd-mirror: allow incomplete demote snapshot to sync after rbd-mirror daemon restart Currently when the rbd-mirror daemon on secondary was killed while a demote snapshot was newly created (0% copied) or partially synced, the image's promotion state was set to **PROMOTION_STATE_ORPHAN** upon restart of rbd-mirror daemon on secondary. This state prevents the demote snapshot sync after restart as bootstrap on secondary fails. In this commit we fix this by assigning promotion state to **PROMOTION_STATE_NON_PRIMARY** for an image with an **incomplete non-primary demote snapshot**. The downside is that if the image is removed on the primary cluster, then after restart of rbd-mirror daemon on secondary cluster, the corresponding image on the secondary also gets removed. This is because deletion propagation is unconditionally enabled precisely for **PROMOTION_STATE_NON_PRIMARY** images and this is okay since the user would have deleted the primary demoted image forcefully. Fixes: https://tracker.ceph.com/issues/73528 Signed-off-by: VinayBhaskar-V (cherry picked from commit 636a3929c1a4052827660f44f35842ac62f6d69a) --- qa/workunits/rbd/rbd_mirror.sh | 57 ++++++++++++++++++++++++++ qa/workunits/rbd/rbd_mirror_helpers.sh | 29 ++++++++++++- src/librbd/mirror/GetInfoRequest.cc | 8 +++- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 61cc0d6c8f3..43deb7baf33 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -526,6 +526,31 @@ for i in ${image2} ${image4}; do remove_image_retry ${CLUSTER2} ${POOL} ${i} done +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: demote image while daemon is offline" + demote_image=test_demote_image + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${demote_image} ${RBD_MIRROR_MODE} + write_image ${CLUSTER2} ${POOL} ${demote_image} 100 + wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${demote_image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${demote_image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+stopped' + stop_mirrors ${CLUSTER1} + write_image ${CLUSTER2} ${POOL} ${demote_image} 100 + demote_image ${CLUSTER2} ${POOL} ${demote_image} + start_mirrors ${CLUSTER1} + wait_for_snapshot_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+unknown' + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + promote_image ${CLUSTER1} ${POOL} ${demote_image} + wait_for_image_replay_started ${CLUSTER2} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+replaying' + remove_image_retry ${CLUSTER1} ${POOL} ${demote_image} +fi + if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then testlog "TEST: request image resync when remote is not primary" test_resync_image=test_resync_image @@ -798,6 +823,38 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries" fi +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: test partially synced demote snapshot sync after daemon restart" + demote_image=test_demote_image + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${demote_image} ${RBD_MIRROR_MODE} 10G + write_image ${CLUSTER2} ${POOL} ${demote_image} 100 + wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${demote_image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${demote_image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+stopped' + write_image ${CLUSTER2} ${POOL} ${demote_image} 2560 4194304 + demote_image ${CLUSTER2} ${POOL} ${demote_image} + get_newest_complete_mirror_snapshot_id ${CLUSTER2} ${POOL} ${demote_image} primary_snap_id + wait_for_non_primary_snap_present ${CLUSTER1} ${POOL} ${demote_image} ${primary_snap_id} + sleep $((RANDOM % 6)) + stop_mirrors ${CLUSTER1} -KILL + SNAPS=$(get_snaps_json ${CLUSTER1} ${POOL} ${demote_image}) + jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' <<< ${SNAPS} + start_mirrors ${CLUSTER1} + wait_for_snapshot_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+unknown' + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${demote_image} + promote_image ${CLUSTER1} ${POOL} ${demote_image} + wait_for_image_replay_started ${CLUSTER2} ${POOL} ${demote_image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${demote_image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${demote_image} 'up+replaying' + remove_image_retry ${CLUSTER1} ${POOL} ${demote_image} + fi +fi + testlog "TEST: force promote with a user snapshot" force_promote_image=test_force_promote_user create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} ${RBD_MIRROR_MODE} 10G diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index e9d4524f58f..6dbd950d460 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -795,6 +795,34 @@ get_newest_mirror_snapshot() ${log} || true } +get_newest_complete_mirror_snapshot_id() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local -n _snap_id=$4 + + _snap_id=$(rbd --cluster ${cluster} snap list --all ${pool}/${image} --format xml | \ + xmlstarlet sel -t -v "(//snapshots/snapshot[namespace/complete='true']/id)[last()]") +} + +wait_for_non_primary_snap_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local primary_snap_id=$4 + local s + + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + rbd --cluster ${cluster} snap list --all ${pool}/${image} --format xml | \ + xmlstarlet sel -t -c "//snapshots/snapshot/namespace[primary_snap_id='${primary_snap_id}']" && return 0 + done + + return 1 +} + wait_for_snapshot_sync_complete() { local local_cluster=$1 @@ -842,7 +870,6 @@ wait_for_replay_complete() fi } - test_status_in_pool_dir() { local cluster=$1 diff --git a/src/librbd/mirror/GetInfoRequest.cc b/src/librbd/mirror/GetInfoRequest.cc index e904a21bba1..40fbfbff9d7 100644 --- a/src/librbd/mirror/GetInfoRequest.cc +++ b/src/librbd/mirror/GetInfoRequest.cc @@ -274,9 +274,15 @@ void GetInfoRequest::calc_promotion_state( *m_primary_mirror_uuid = mirror_ns->primary_mirror_uuid; break; case cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED: - case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED: *m_promotion_state = PROMOTION_STATE_ORPHAN; break; + case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED: + if (mirror_ns->complete) { + *m_promotion_state = PROMOTION_STATE_ORPHAN; + } else { + *m_promotion_state = PROMOTION_STATE_NON_PRIMARY; + } + break; } break; } -- 2.39.5