From 44c116dfd6f519538b6c1401e3c3bc283d31b78a Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Mon, 9 Jan 2017 10:40:46 +0100 Subject: [PATCH] rbd-mirror: check image mirroring state when bootstrapping Fixes: http://tracker.ceph.com/issues/18447 Signed-off-by: Mykola Golub (cherry picked from commit 5fc5a8ac895524f05eed6e7db20b0dda3a8cb60f) --- src/test/rbd_mirror/test_ImageReplayer.cc | 55 ++++++++++++++++++- .../image_replayer/IsPrimaryRequest.cc | 50 ++++++++++++++++- .../image_replayer/IsPrimaryRequest.h | 14 ++++- 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 9916e362e57c3..5bd6c53a46980 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -99,6 +99,7 @@ public: EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), m_remote_ioctx)); + EXPECT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_POOL)); m_image_name = get_temp_image_name(); uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); @@ -392,7 +393,7 @@ TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) TEST_F(TestImageReplayer, BootstrapErrorNoJournal) { - // disable remote journal journaling + // disable remote image journaling librbd::ImageCtx *ictx; open_remote_image(&ictx); uint64_t features; @@ -407,6 +408,58 @@ TEST_F(TestImageReplayer, BootstrapErrorNoJournal) ASSERT_EQ(-ENOENT, cond.wait()); } +TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled) +{ + // disable remote image mirroring + ASSERT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::mirror_image_disable(ictx, true)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapMirrorDisabling) +{ + // set remote image mirroring state to DISABLING + ASSERT_EQ(0, librbd::mirror_mode_set(m_remote_ioctx, RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::mirror_image_enable(ictx)); + cls::rbd::MirrorImage mirror_image; + ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id, + &mirror_image)); + mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING; + ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx, ictx->id, + mirror_image)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + ASSERT_TRUE(m_replayer->is_stopped()); +} + +TEST_F(TestImageReplayer, BootstrapDemoted) +{ + // demote remote image + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::mirror_image_demote(ictx)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + ASSERT_TRUE(m_replayer->is_stopped()); +} + TEST_F(TestImageReplayer, StartInterrupted) { create_replayer<>(); diff --git a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc index a9169f0808a1c..8a45a95081bef 100644 --- a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.cc @@ -4,6 +4,7 @@ #include "IsPrimaryRequest.h" #include "common/errno.h" #include "common/WorkQueue.h" +#include "cls/rbd/cls_rbd_client.h" #include "librbd/ImageCtx.h" #include "librbd/Journal.h" #include "librbd/Utils.h" @@ -20,6 +21,7 @@ namespace mirror { namespace image_replayer { using librbd::util::create_context_callback; +using librbd::util::create_rados_ack_callback; template IsPrimaryRequest::IsPrimaryRequest(I *image_ctx, bool *primary, @@ -29,7 +31,53 @@ IsPrimaryRequest::IsPrimaryRequest(I *image_ctx, bool *primary, template void IsPrimaryRequest::send() { - send_is_tag_owner(); + send_get_mirror_state(); +} + +template +void IsPrimaryRequest::send_get_mirror_state() { + dout(20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, m_image_ctx->id); + + librados::AioCompletion *aio_comp = create_rados_ack_callback< + IsPrimaryRequest, &IsPrimaryRequest::handle_get_mirror_state>(this); + int r = m_image_ctx->md_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, + &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void IsPrimaryRequest::handle_get_mirror_state(int r) { + dout(20) << ": r=" << r << dendl; + + cls::rbd::MirrorImage mirror_image; + if (r == 0) { + bufferlist::iterator iter = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + if (r == 0) { + if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + send_is_tag_owner(); + return; + } else if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + dout(5) << ": image mirroring is being disabled" << dendl; + *m_primary = false; + } else { + derr << ": image mirroring is disabled" << dendl; + r = -EINVAL; + } + } else { + derr << ": failed to decode image mirror state: " << cpp_strerror(r) + << dendl; + } + } else { + derr << ": failed to retrieve image mirror state: " << cpp_strerror(r) + << dendl; + } + + finish(r); } template diff --git a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h index e3482084b8a27..ddb332cbfe69c 100644 --- a/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h +++ b/src/tools/rbd_mirror/image_replayer/IsPrimaryRequest.h @@ -4,6 +4,8 @@ #ifndef RBD_MIRROR_IMAGE_REPLAYER_IS_PRIMARY_REQUEST_H #define RBD_MIRROR_IMAGE_REPLAYER_IS_PRIMARY_REQUEST_H +#include "include/buffer.h" + class Context; class ContextWQ; namespace librbd { class ImageCtx; } @@ -31,8 +33,11 @@ private: * * | * v - * IS_TAG_OWNER * * * * * * * - * | * (error) + * GET_MIRROR_STATE * * * * * + * | * + * v * + * IS_TAG_OWNER * * * * * * * (error) + * | * * v * * < * * * * * * * * * @@ -42,6 +47,11 @@ private: bool *m_primary; Context *m_on_finish; + bufferlist m_out_bl; + + void send_get_mirror_state(); + void handle_get_mirror_state(int r); + void send_is_tag_owner(); void handle_is_tag_owner(int r); -- 2.39.5