From: Jason Dillaman Date: Fri, 20 Dec 2019 03:43:06 +0000 (-0500) Subject: rbd-mirror: reset bootstrap state to replaying if local image promoted X-Git-Tag: v15.1.0~352^2~9 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8c33391a6294f68b590f26fff190a34a2dc317eb;p=ceph.git rbd-mirror: reset bootstrap state to replaying if local image promoted The previous logic had a race in that it would never run if the local image was force-promoted or if the remote image was demoted and the local image was promoted before the rbd-mirror daemon could catch the change of state. This change resets the mirror peer state from syncing to replaying if the local image is no longer connected to the remote image. This prevents an unexpected (and corrupted) re-sync from the remote. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 4a9a1749daa7..45610f125b0b 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -498,7 +498,7 @@ public: bool m_do_resync; }; -TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { +TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemotedRemoteSyncingState) { create_local_image(); InSequence seq; @@ -515,10 +515,21 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { expect_get_remote_mirror_info(mock_get_mirror_info_request, {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, "uuid", cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, - librbd::mirror::PROMOTION_STATE_NON_PRIMARY, 0); + librbd::mirror::PROMOTION_STATE_PRIMARY, 0); - // switch the state to replaying + // open the local image librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // update client state librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -527,6 +538,40 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { ::journal::MockJournaler mock_journaler; expect_journaler_update_client(mock_journaler, client_data, 0); + // lookup remote image tag class + client_data = {librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + MockCloseImageRequest mock_close_image_request; expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); @@ -541,7 +586,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { "local mirror uuid", "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); - ASSERT_EQ(-EREMOTEIO, ctx.wait()); + ASSERT_EQ(0, ctx.wait()); } TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteNotTagOwner) { diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 98e891433405..2542356eae65 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -175,23 +175,14 @@ void BootstrapRequest::handle_get_remote_mirror_info(int r) { return; } - if (m_promotion_state != librbd::mirror::PROMOTION_STATE_PRIMARY) { - if (m_local_image_id.empty()) { - // no local image and remote isn't primary -- don't sync it - dout(5) << "remote image is not primary -- not syncing" - << dendl; - m_ret_val = -EREMOTEIO; - close_remote_image(); - return; - } else if (m_client_meta->state != - librbd::journal::MIRROR_PEER_STATE_REPLAYING) { - // ensure we attempt to re-sync to remote if it's re-promoted - dout(5) << "remote image is not primary -- sync interrupted" - << dendl; - m_ret_val = -EREMOTEIO; - update_client_state(); - return; - } + if (m_promotion_state != librbd::mirror::PROMOTION_STATE_PRIMARY && + m_local_image_id.empty()) { + // no local image and remote isn't primary -- don't sync it + dout(5) << "remote image is not primary -- not syncing" + << dendl; + m_ret_val = -EREMOTEIO; + close_remote_image(); + return; } if (!m_client_meta->image_id.empty()) { @@ -209,36 +200,6 @@ void BootstrapRequest::handle_get_remote_mirror_info(int r) { open_local_image(); } -template -void BootstrapRequest::update_client_state() { - dout(15) << dendl; - update_progress("UPDATE_CLIENT_STATE"); - - librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); - client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - - librbd::journal::ClientData client_data(client_meta); - bufferlist data_bl; - encode(client_data, data_bl); - - Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_update_client_state>( - this); - m_journaler->update_client(data_bl, ctx); -} - -template -void BootstrapRequest::handle_update_client_state(int r) { - dout(15) << "r=" << r << dendl; - if (r < 0) { - derr << "failed to update client: " << cpp_strerror(r) << dendl; - } else { - m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - } - - close_remote_image(); -} - template void BootstrapRequest::open_local_image() { dout(15) << "local_image_id=" << m_local_image_id << dendl; @@ -331,6 +292,47 @@ void BootstrapRequest::handle_open_local_image(int r) { return; } + update_client_state(); +} + +template +void BootstrapRequest::update_client_state() { + if (m_client_meta->state != librbd::journal::MIRROR_PEER_STATE_SYNCING || + m_local_tag_data.mirror_uuid == m_remote_mirror_uuid) { + get_remote_tag_class(); + return; + } + + // our local image is not primary, is flagged as syncing on the remote side, + // but is no longer tied to the remote -- this implies we were forced + // promoted and then demoted at some point + dout(15) << dendl; + update_progress("UPDATE_CLIENT_STATE"); + + librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); + client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data(client_meta); + bufferlist data_bl; + encode(client_data, data_bl); + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_update_client_state>( + this); + m_journaler->update_client(data_bl, ctx); +} + +template +void BootstrapRequest::handle_update_client_state(int r) { + dout(15) << "r=" << r << dendl; + if (r < 0) { + derr << "failed to update client: " << cpp_strerror(r) << dendl; + m_ret_val = r; + close_local_image(); + return; + } + + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; get_remote_tag_class(); } diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index 02c4296753f1..c2608caa4702 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -112,31 +112,31 @@ private: * | v * * | * | (remote image primary) * * | * \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * * * | - * | | . * * | - * | | . (image doesn't exist) * * | - * | | . . > UNREGISTER_CLIENT * * * * * * * | - * | | | * * | - * | | v * * | - * | | REGISTER_CLIENT * * * * * * * * | - * | | | * * | - * | | \-----------------------*---*---/ - * | | * * - * | v (skip if not needed) * * - * | GET_REMOTE_TAG_CLASS * * * * * * * - * | | * * * - * | v (skip if not needed) * * * - * | GET_REMOTE_TAGS * * * * * * * * * - * | | * * * - * | v (skip if not needed) v * * - * | IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE * * - * | | | * * - * | \-----------------\ /-----/ * * - * | | * * - * | | * * - * | (skip if not needed) | * * - * \----> UPDATE_CLIENT_STATE *|* * * * * * * * * * * - * | | * * - * /-----------/----------------/ * * + * | . * * | + * | . (image doesn't exist) * * | + * | . . > UNREGISTER_CLIENT * * * * * * * | + * | | * * | + * | v * * | + * | REGISTER_CLIENT * * * * * * * * | + * | | * * | + * | \-----------------------*---*---/ + * | * * + * v (skip if not needed) * * + * UPDATE_CLIENT_STATE * * + * | * * + * v (skip if not needed) * * + * GET_REMOTE_TAG_CLASS * * * * * * * + * | * * * + * v (skip if not needed) * * * + * GET_REMOTE_TAGS * * * * * * * * * + * | * * * + * v (skip if not needed) v * * + * IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE * * + * | | * * + * \-----------------\ /-----/ * * + * | * * + * | * * + * /----------------------------/ * * * | * * * v * * * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * * * * * * * @@ -188,9 +188,6 @@ private: void get_remote_mirror_info(); void handle_get_remote_mirror_info(int r); - void update_client_state(); - void handle_update_client_state(int r); - void open_local_image(); void handle_open_local_image(int r); @@ -206,6 +203,9 @@ private: void update_client_image(); void handle_update_client_image(int r); + void update_client_state(); + void handle_update_client_state(int r); + void get_remote_tag_class(); void handle_get_remote_tag_class(int r);