From: Jason Dillaman Date: Wed, 19 Jul 2017 18:24:36 +0000 (-0400) Subject: rbd-mirror: clean up bootstrap recovery from missing local image X-Git-Tag: v13.0.0~218^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2a6cfd83737ac2cb80d0c2a735985cc2a9b6f871;p=ceph.git rbd-mirror: clean up bootstrap recovery from missing local image Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index c7b995387e1d..4ed498370a70 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -325,6 +325,14 @@ public: }))); } + void expect_journaler_unregister_client(::journal::MockJournaler &mock_journaler, + int r) { + EXPECT_CALL(mock_journaler, unregister_client(_)) + .WillOnce(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })); + } + void expect_journaler_update_client(::journal::MockJournaler &mock_journaler, const librbd::journal::ClientData &client_data, int r) { @@ -468,6 +476,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -479,12 +494,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, false, 0); @@ -521,6 +531,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -532,12 +549,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -595,6 +607,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -606,12 +625,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -679,6 +693,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -690,12 +711,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -751,6 +767,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -762,12 +785,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -823,6 +841,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ @@ -834,12 +859,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -883,6 +903,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal client = {}; expect_journaler_get_client(mock_journaler, "local mirror uuid", @@ -893,12 +920,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { client_data.client_meta = mirror_peer_client_meta; expect_journaler_register_client(mock_journaler, client_data, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -955,6 +977,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { librbd::Journal<>::IMAGE_CLIENT_ID, client, 0); + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // lookup local peer in remote journal librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ "missing image id"}; @@ -965,12 +994,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { expect_journaler_get_client(mock_journaler, "local mirror uuid", client, 0); - // open the remote image - librbd::MockJournal mock_journal; - librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); - MockOpenImageRequest mock_open_image_request; - expect_open_image(mock_open_image_request, m_remote_io_ctx, - mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -979,6 +1003,15 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, "missing image id", nullptr, -ENOENT); + // re-register the client + expect_journaler_unregister_client(mock_journaler, 0); + mirror_peer_client_meta = {}; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, 0); + + // test if remote image is primary + expect_is_primary(mock_is_primary_request, true, 0); + // create the missing local image librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 434aa783c050..8fdf1b270b9e 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -143,6 +143,35 @@ void BootstrapRequest::handle_get_remote_tag_class(int r) { m_remote_tag_class = client_meta->tag_class; dout(10) << ": remote tag class=" << m_remote_tag_class << dendl; + open_remote_image(); +} + +template +void BootstrapRequest::open_remote_image() { + dout(20) << dendl; + + update_progress("OPEN_REMOTE_IMAGE"); + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_open_remote_image>( + this); + OpenImageRequest *request = OpenImageRequest::create( + m_remote_io_ctx, &m_remote_image_ctx, m_remote_image_id, false, + ctx); + request->send(); +} + +template +void BootstrapRequest::handle_open_remote_image(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": failed to open remote image: " << cpp_strerror(r) << dendl; + assert(m_remote_image_ctx == nullptr); + finish(r); + return; + } + get_client(); } @@ -166,11 +195,12 @@ void BootstrapRequest::handle_get_client(int r) { dout(10) << ": client not registered" << dendl; } else if (r < 0) { derr << ": failed to retrieve client: " << cpp_strerror(r) << dendl; - finish(r); + m_ret_val = r; + close_remote_image(); return; } else if (decode_client_meta()) { // skip registration if it already exists - open_remote_image(); + is_primary(); return; } @@ -202,40 +232,12 @@ void BootstrapRequest::handle_register_client(int r) { if (r < 0) { derr << ": failed to register with remote journal: " << cpp_strerror(r) << dendl; - finish(r); + m_ret_val = r; + close_remote_image(); return; } *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id); - open_remote_image(); -} - -template -void BootstrapRequest::open_remote_image() { - dout(20) << dendl; - - update_progress("OPEN_REMOTE_IMAGE"); - - Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_open_remote_image>( - this); - OpenImageRequest *request = OpenImageRequest::create( - m_remote_io_ctx, &m_remote_image_ctx, m_remote_image_id, false, - ctx); - request->send(); -} - -template -void BootstrapRequest::handle_open_remote_image(int r) { - dout(20) << ": r=" << r << dendl; - - if (r < 0) { - derr << ": failed to open remote image: " << cpp_strerror(r) << dendl; - assert(m_remote_image_ctx == nullptr); - finish(r); - return; - } - is_primary(); } @@ -311,7 +313,7 @@ void BootstrapRequest::handle_update_client_state(int r) { if (r < 0) { derr << ": failed to update client: " << cpp_strerror(r) << dendl; } else { - m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;; + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; } close_remote_image(); @@ -339,7 +341,7 @@ void BootstrapRequest::handle_open_local_image(int r) { if (r == -ENOENT) { assert(*m_local_image_ctx == nullptr); dout(10) << ": local image missing" << dendl; - create_local_image(); + unregister_client(); return; } else if (r == -EREMOTEIO) { assert(*m_local_image_ctx == nullptr); @@ -390,6 +392,33 @@ void BootstrapRequest::handle_open_local_image(int r) { update_client_image(); } +template +void BootstrapRequest::unregister_client() { + dout(20) << dendl; + update_progress("UNREGISTER_CLIENT"); + + m_local_image_id = ""; + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_unregister_client>( + this); + m_journaler->unregister_client(ctx); +} + +template +void BootstrapRequest::handle_unregister_client(int r) { + dout(20) << ": r=" << r << dendl; + if (r < 0) { + derr << ": failed to unregister with remote journal: " << cpp_strerror(r) + << dendl; + m_ret_val = r; + close_remote_image(); + return; + } + + *m_client_meta = librbd::journal::MirrorPeerClientMeta(""); + register_client(); +} + template void BootstrapRequest::create_local_image() { dout(20) << dendl; @@ -426,10 +455,6 @@ void BootstrapRequest::handle_create_local_image(int r) { template void BootstrapRequest::update_client_image() { - dout(20) << dendl; - - update_progress("UPDATE_CLIENT_IMAGE"); - if (m_client_meta->image_id == (*m_local_image_ctx)->id) { // already registered local image with remote journal get_remote_tags(); @@ -438,6 +463,7 @@ void BootstrapRequest::update_client_image() { m_local_image_id = (*m_local_image_ctx)->id; dout(20) << dendl; + update_progress("UPDATE_CLIENT_IMAGE"); librbd::journal::MirrorPeerClientMeta client_meta{m_local_image_id}; client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; @@ -459,14 +485,14 @@ void BootstrapRequest::handle_update_client_image(int r) { if (r < 0) { derr << ": failed to update client: " << cpp_strerror(r) << dendl; m_ret_val = r; - close_local_image(); + close_remote_image(); return; } if (m_canceled) { dout(10) << ": request canceled" << dendl; m_ret_val = -ECANCELED; - close_local_image(); + close_remote_image(); return; } diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index cebd568972ba..668dd8375d4b 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -92,57 +92,55 @@ private: * * | * v - * GET_REMOTE_TAG_CLASS * * * * * * * * * * * * * * * * - * | * - * v * - * GET_CLIENT * * * * * * * * * * * * * * * * * * * * * - * | * - * v (skip if not needed) * (error) - * REGISTER_CLIENT * * * * * * * * * * * * * * * * * * - * | * - * v * - * OPEN_REMOTE_IMAGE * * * * * * * * * * * * * * * * * - * | * - * v * - * IS_PRIMARY * * * * * * * * * * * * * * * * * * * * * - * | * - * | (remote image primary) * - * \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * * - * | | . ^ * - * | | . | * - * | | . \-----------------------\ * - * | | . | * - * | | . (image sync requested) | * - * | | . . > REMOVE_LOCAL_IMAGE * * * * * - * | | . | | * - * | | . (image doesn't | | * - * | | . exist) v | * - * | | . . > CREATE_LOCAL_IMAGE * * * * * - * | | | | * - * | | \-----------------/ * - * | | * - * | v (skip if not needed) * - * | UPDATE_CLIENT_IMAGE * * * * * * - * | | * * - * | v (skip if not needed) * * - * | GET_REMOTE_TAGS * * * * * * * * - * | | * * - * | v (skip if not needed) v * - * | IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE * - * | | | * - * | \-----------------\ /-----/ * - * | | * - * | | * - * | (skip if not needed) | * - * \----> UPDATE_CLIENT_STATE *|* * * * * * * * * * - * | | * - * /-----------/----------------/ * - * | * - * v * - * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * * * * * * - * | - * v - * + * GET_REMOTE_TAG_CLASS * * * * * * * * * * * * * * * * * * + * | * (error) + * v * + * OPEN_REMOTE_IMAGE * * * * * * * * * * * * * * * * * * * + * | * + * v * + * GET_CLIENT * * * * * * * * * * * * * * * * * * * * * * + * | * * + * |/----------------------------------------------*---*---\ + * v (skip if not needed) * * | + * REGISTER_CLIENT * * * * * * * * * * * * * * * * * * * | + * | * * | + * v * * | + * IS_PRIMARY * * * * * * * * * * * * * * * * * * * * * * | + * | * * | + * | (remote image primary, no local image id) * * | + * \----> CREATE_LOCAL_IMAGE * * * * * * * * * * * * * | + * | | * * | + * | v * * | + * | (remote image primary) * * | + * \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * * * | + * | | . * * | + * | | . (image doesn't exist) * * | + * | | . . > UNREGISTER_CLIENT * * * * * * * | + * | | | * * | + * | | \-----------------------*---*---/ + * | | * * + * | v (skip if not needed) * * + * | UPDATE_CLIENT_IMAGE * * * * * * * + * | | * * * + * | v (skip if not needed) * * * + * | GET_REMOTE_TAGS * * * * * * * * * + * | | * * * + * | v (skip if not needed) v * * + * | IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE * * + * | | | * * + * | \-----------------\ /-----/ * * + * | | * * + * | | * * + * | (skip if not needed) | * * + * \----> UPDATE_CLIENT_STATE *|* * * * * * * * * * * + * | | * * + * /-----------/----------------/ * * + * | * * + * v * * + * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * * * * * * * + * | * + * v * + * < * * * * * * * * * * * * * * * * * * * * * * * * * @endverbatim */ @@ -199,8 +197,8 @@ private: void open_local_image(); void handle_open_local_image(int r); - void remove_local_image(); - void handle_remove_local_image(int r); + void unregister_client(); + void handle_unregister_client(int r); void create_local_image(); void handle_create_local_image(int r);