From: Jason Dillaman Date: Mon, 9 Mar 2020 21:04:27 +0000 (-0400) Subject: librbd: enable/disable implicit non-primary feature bit X-Git-Tag: v15.1.1~31^2~10 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=814c13a8cfb12568baa5c6978965edd68c908b1a;p=ceph.git librbd: enable/disable implicit non-primary feature bit When promoted to primary, disable the non-primary feature bit and when demoted (or created non-primary), enable the non-primary feature bit. This will prevent all non rbd-mirror RBD clients from modifying the RBD image. Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 849aeacfd3d88..ebdaa4926cbac 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -480,6 +480,14 @@ int Mirror::image_disable(I *ictx, bool force) { bool rollback = false; BOOST_SCOPE_EXIT_ALL(ictx, &mirror_image_internal, &rollback) { if (rollback) { + // restore the mask bit for treating the non-primary feature as read-only + ictx->image_lock.lock(); + ictx->read_only_mask |= IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + ictx->image_lock.unlock(); + + ictx->state->handle_update_notification(); + + // attempt to restore the image state CephContext *cct = ictx->cct; mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED; int r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id, @@ -541,6 +549,17 @@ int Mirror::image_disable(I *ictx, bool force) { image_locker.unlock(); if (mirror_image_internal.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + // don't let the non-primary feature bit prevent image updates + ictx->image_lock.lock(); + ictx->read_only_mask &= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + ictx->image_lock.unlock(); + + r = ictx->state->refresh(); + if (r < 0) { + rollback = true; + return r; + } + // remove any snapshot-based mirroring image-meta from image std::string mirror_uuid; r = uuid_get(ictx->md_ctx, &mirror_uuid); @@ -603,21 +622,31 @@ void Mirror::image_promote(I *ictx, bool force, Context *on_finish) { ldout(cct, 20) << "ictx=" << ictx << ", " << "force=" << force << dendl; - auto on_refresh = new LambdaContext([ictx, force, on_finish](int r) { + // don't let the non-primary feature bit prevent image updates + ictx->image_lock.lock(); + ictx->read_only_mask &= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + ictx->image_lock.unlock(); + + auto on_promote = new LambdaContext([ictx, on_finish](int r) { + ictx->image_lock.lock(); + ictx->read_only_mask |= IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + ictx->image_lock.unlock(); + + ictx->state->handle_update_notification(); + on_finish->complete(r); + }); + + auto on_refresh = new LambdaContext([ictx, force, on_promote](int r) { if (r < 0) { lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; - on_finish->complete(r); + on_promote->complete(r); return; } - auto req = mirror::PromoteRequest<>::create(*ictx, force, on_finish); + auto req = mirror::PromoteRequest<>::create(*ictx, force, on_promote); req->send(); }); - if (ictx->state->is_refresh_required()) { - ictx->state->refresh(on_refresh); - } else { - on_refresh->complete(0); - } + ictx->state->refresh(on_refresh); } template diff --git a/src/librbd/mirror/EnableRequest.cc b/src/librbd/mirror/EnableRequest.cc index beee05ac4e844..c2cbbbd322c7f 100644 --- a/src/librbd/mirror/EnableRequest.cc +++ b/src/librbd/mirror/EnableRequest.cc @@ -105,10 +105,12 @@ void EnableRequest::handle_get_mirror_image(int r) { template void EnableRequest::get_tag_owner() { - if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT || - !m_non_primary_global_image_id.empty()) { + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { create_primary_snapshot(); return; + } else if (!m_non_primary_global_image_id.empty()) { + image_state_update(); + return; } ldout(m_cct, 10) << dendl; @@ -143,7 +145,8 @@ void EnableRequest::handle_get_tag_owner(int r) { template void EnableRequest::create_primary_snapshot() { if (!m_non_primary_global_image_id.empty()) { - image_state_update(); + // special case for rbd-mirror creating a non-primary image + enable_non_primary_feature(); return; } @@ -163,6 +166,50 @@ template void EnableRequest::handle_create_primary_snapshot(int r) { ldout(m_cct, 10) << "r=" << r << dendl; + if (r < 0) { + lderr(m_cct) << "failed to create initial primary snapshot: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + image_state_update(); +} + +template +void EnableRequest::enable_non_primary_feature() { + if (m_mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + image_state_update(); + return; + } + + ldout(m_cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + EnableRequest, + &EnableRequest::handle_enable_non_primary_feature>(this); + int r = m_io_ctx.aio_operate(util::header_name(m_image_id), aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void EnableRequest::handle_enable_non_primary_feature(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + image_state_update(); } diff --git a/src/librbd/mirror/EnableRequest.h b/src/librbd/mirror/EnableRequest.h index 853e96a7efca3..c9945d7b56c7d 100644 --- a/src/librbd/mirror/EnableRequest.h +++ b/src/librbd/mirror/EnableRequest.h @@ -55,6 +55,9 @@ private: * v (skip if not needed) * * CREATE_PRIMARY_SNAPSHOT * * * * | * + * v (skip if not needed) * + * ENABLE_NON_PRIMARY_FEATURE * + * | * * v * * IMAGE_STATE_UPDATE * * * * * * * | * @@ -93,6 +96,9 @@ private: void create_primary_snapshot(); void handle_create_primary_snapshot(int r); + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + void image_state_update(); void handle_image_state_update(int r); diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc index 43d555990b1ef..23cbd1d479766 100644 --- a/src/librbd/mirror/snapshot/DemoteRequest.cc +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -21,9 +21,45 @@ namespace mirror { namespace snapshot { using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; template void DemoteRequest::send() { + enable_non_primary_feature(); +} + +template +void DemoteRequest::enable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + DemoteRequest, + &DemoteRequest::handle_enable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void DemoteRequest::handle_enable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + create_snapshot(); } diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h index 94a2eb333e07d..63c9356458dc2 100644 --- a/src/librbd/mirror/snapshot/DemoteRequest.h +++ b/src/librbd/mirror/snapshot/DemoteRequest.h @@ -42,6 +42,9 @@ private: * * | * v + * ENABLE_NON_PRIMARY_FEATURE + * | + * v * CREATE_SNAPSHOT * | * v @@ -54,6 +57,9 @@ private: std::string m_global_image_id; Context *m_on_finish; + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + void create_snapshot(); void handle_create_snapshot(int r); diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc index d02959077931c..ef62c0813960b 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.cc +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -27,6 +27,7 @@ namespace snapshot { using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; template void PromoteRequest::send() { @@ -312,6 +313,40 @@ void PromoteRequest::handle_create_promote_snapshot(int r) { return; } + disable_non_primary_feature(); +} + +template +void PromoteRequest::disable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // remove the non-primary feature flag so that the image can be + // R/W by standard RBD clients + librados::ObjectWriteOperation op; + cls_client::set_features(&op, 0U, RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + PromoteRequest, + &PromoteRequest::handle_disable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void PromoteRequest::handle_disable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to disable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + release_exclusive_lock(); } diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h index 1583551f1d534..f24138ff33863 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.h +++ b/src/librbd/mirror/snapshot/PromoteRequest.h @@ -66,6 +66,9 @@ private: * CREATE_PROMOTE_SNAPSHOT <--------------------/ * | * v + * DISABLE_NON_PRIMARY_FEATURE + * | + * v * RELEASE_EXCLUSIVE_LOCK (skip if not needed) * | * v @@ -128,6 +131,9 @@ private: void create_promote_snapshot(); void handle_create_promote_snapshot(int r); + void disable_non_primary_feature(); + void handle_disable_non_primary_feature(int r); + void release_exclusive_lock(); void handle_release_exclusive_lock(int r);