librbd: enable/disable implicit non-primary feature bit

When promoted to primary, disable the non-primary feature bit and
when demoted (or created non-primary), enable the non-primary feature
bit. This will prevent all non rbd-mirror RBD clients from modifying
the RBD image.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc
index 849aeacfd3d885814eb86b437bb63398e403edda..ebdaa4926cbac62b9b9c0662f258cd42fe57f4b2 100644 (file)
--- a/src/librbd/api/Mirror.cc
+++ b/src/librbd/api/Mirror.cc
@@ -480,6 +480,14 @@ int Mirror<I>::image_disable(I *ictx, bool force) {
   bool rollback = false;
   BOOST_SCOPE_EXIT_ALL(ictx, &mirror_image_internal, &rollback) {
     if (rollback) {
+      // restore the mask bit for treating the non-primary feature as read-only
+      ictx->image_lock.lock();
+      ictx->read_only_mask |= IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+      ictx->image_lock.unlock();
+
+      ictx->state->handle_update_notification();
+
+      // attempt to restore the image state
       CephContext *cct = ictx->cct;
       mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED;
       int r = cls_client::mirror_image_set(&ictx->md_ctx, ictx->id,
@@ -541,6 +549,17 @@ int Mirror<I>::image_disable(I *ictx, bool force) {
   image_locker.unlock();
 
   if (mirror_image_internal.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+    // don't let the non-primary feature bit prevent image updates
+    ictx->image_lock.lock();
+    ictx->read_only_mask &= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+    ictx->image_lock.unlock();
+
+    r = ictx->state->refresh();
+    if (r < 0) {
+      rollback = true;
+      return r;
+    }
+
     // remove any snapshot-based mirroring image-meta from image
     std::string mirror_uuid;
     r = uuid_get(ictx->md_ctx, &mirror_uuid);
@@ -603,21 +622,31 @@ void Mirror<I>::image_promote(I *ictx, bool force, Context *on_finish) {
   ldout(cct, 20) << "ictx=" << ictx << ", "
                  << "force=" << force << dendl;
 
-  auto on_refresh = new LambdaContext([ictx, force, on_finish](int r) {
+  // don't let the non-primary feature bit prevent image updates
+  ictx->image_lock.lock();
+  ictx->read_only_mask &= ~IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+  ictx->image_lock.unlock();
+
+  auto on_promote = new LambdaContext([ictx, on_finish](int r) {
+      ictx->image_lock.lock();
+      ictx->read_only_mask |= IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+      ictx->image_lock.unlock();
+
+      ictx->state->handle_update_notification();
+      on_finish->complete(r);
+    });
+
+  auto on_refresh = new LambdaContext([ictx, force, on_promote](int r) {
       if (r < 0) {
         lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl;
-        on_finish->complete(r);
+        on_promote->complete(r);
         return;
       }
 
-      auto req = mirror::PromoteRequest<>::create(*ictx, force, on_finish);
+      auto req = mirror::PromoteRequest<>::create(*ictx, force, on_promote);
       req->send();
     });
-  if (ictx->state->is_refresh_required()) {
-    ictx->state->refresh(on_refresh);
-  } else {
-    on_refresh->complete(0);
-  }
+  ictx->state->refresh(on_refresh);
 }
 
 template <typename I>

diff --git a/src/librbd/mirror/EnableRequest.cc b/src/librbd/mirror/EnableRequest.cc
index beee05ac4e8443f4f5fc391ecdec2ce5a3c58688..c2cbbbd322c7facfc1f5b453437bac3fe26de450 100644 (file)
--- a/src/librbd/mirror/EnableRequest.cc
+++ b/src/librbd/mirror/EnableRequest.cc
@@ -105,10 +105,12 @@ void EnableRequest<I>::handle_get_mirror_image(int r) {
 
 template <typename I>
 void EnableRequest<I>::get_tag_owner() {
-  if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT ||
-      !m_non_primary_global_image_id.empty()) {
+  if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
     create_primary_snapshot();
     return;
+  } else if (!m_non_primary_global_image_id.empty()) {
+    image_state_update();
+    return;
   }
 
   ldout(m_cct, 10)  << dendl;
@@ -143,7 +145,8 @@ void EnableRequest<I>::handle_get_tag_owner(int r) {
 template <typename I>
 void EnableRequest<I>::create_primary_snapshot() {
   if (!m_non_primary_global_image_id.empty()) {
-    image_state_update();
+    // special case for rbd-mirror creating a non-primary image
+    enable_non_primary_feature();
     return;
   }
 
@@ -163,6 +166,50 @@ template <typename I>
 void EnableRequest<I>::handle_create_primary_snapshot(int r) {
   ldout(m_cct, 10) << "r=" << r << dendl;
 
+  if (r < 0) {
+    lderr(m_cct) << "failed to create initial primary snapshot: "
+                 << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  image_state_update();
+}
+
+template <typename I>
+void EnableRequest<I>::enable_non_primary_feature() {
+  if (m_mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+    image_state_update();
+    return;
+  }
+
+  ldout(m_cct, 10) << dendl;
+
+  // ensure image is flagged with non-primary feature so that
+  // standard RBD clients cannot write to it.
+  librados::ObjectWriteOperation op;
+  cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY,
+                           RBD_FEATURE_NON_PRIMARY);
+
+  auto aio_comp = create_rados_callback<
+    EnableRequest<I>,
+    &EnableRequest<I>::handle_enable_non_primary_feature>(this);
+  int r = m_io_ctx.aio_operate(util::header_name(m_image_id), aio_comp, &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void EnableRequest<I>::handle_enable_non_primary_feature(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to enable non-primary feature: "
+                 << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
   image_state_update();
 }
 

diff --git a/src/librbd/mirror/EnableRequest.h b/src/librbd/mirror/EnableRequest.h
index 853e96a7efca37c478ea1e6bc98250c5f8ff9ccd..c9945d7b56c7dcd90dc1cae787df8e29d72446be 100644 (file)
--- a/src/librbd/mirror/EnableRequest.h
+++ b/src/librbd/mirror/EnableRequest.h
@@ -55,6 +55,9 @@ private:
    *    v (skip if not needed)    *
    * CREATE_PRIMARY_SNAPSHOT  * * *
    *    |                         *
+   *    v (skip if not needed)    *
+   * ENABLE_NON_PRIMARY_FEATURE   *
+   *    |                         *
    *    v                         *
    * IMAGE_STATE_UPDATE * * * * * *
    *    |                         *
@@ -93,6 +96,9 @@ private:
   void create_primary_snapshot();
   void handle_create_primary_snapshot(int r);
 
+  void enable_non_primary_feature();
+  void handle_enable_non_primary_feature(int r);
+
   void image_state_update();
   void handle_image_state_update(int r);
 

diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc
index 43d555990b1ef9155fca15921bca0d1f81d049fb..23cbd1d479766f4e455dd63f7601decad275d9be 100644 (file)
--- a/src/librbd/mirror/snapshot/DemoteRequest.cc
+++ b/src/librbd/mirror/snapshot/DemoteRequest.cc
@@ -21,9 +21,45 @@ namespace mirror {
 namespace snapshot {
 
 using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
 
 template <typename I>
 void DemoteRequest<I>::send() {
+  enable_non_primary_feature();
+}
+
+template <typename I>
+void DemoteRequest<I>::enable_non_primary_feature() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << dendl;
+
+  // ensure image is flagged with non-primary feature so that
+  // standard RBD clients cannot write to it.
+  librados::ObjectWriteOperation op;
+  cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY,
+                           RBD_FEATURE_NON_PRIMARY);
+
+  auto aio_comp = create_rados_callback<
+    DemoteRequest<I>,
+    &DemoteRequest<I>::handle_enable_non_primary_feature>(this);
+  int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp,
+                                          &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void DemoteRequest<I>::handle_enable_non_primary_feature(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to enable non-primary feature: "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
   create_snapshot();
 }
 

diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h
index 94a2eb333e07d9b8ae8078e5939387bd8b95e40b..63c9356458dc23f28c35c3cbe42c52d406997c8f 100644 (file)
--- a/src/librbd/mirror/snapshot/DemoteRequest.h
+++ b/src/librbd/mirror/snapshot/DemoteRequest.h
@@ -42,6 +42,9 @@ private:
    * <start>
    *    |
    *    v
+   * ENABLE_NON_PRIMARY_FEATURE
+   *    |
+   *    v
    * CREATE_SNAPSHOT
    *    |
    *    v
@@ -54,6 +57,9 @@ private:
   std::string m_global_image_id;
   Context *m_on_finish;
 
+  void enable_non_primary_feature();
+  void handle_enable_non_primary_feature(int r);
+
   void create_snapshot();
   void handle_create_snapshot(int r);
 

diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc
index d02959077931ccec630e9672850aaeb6f3c32740..ef62c0813960bb285c054656a0f91bc5e6c276dc 100644 (file)
--- a/src/librbd/mirror/snapshot/PromoteRequest.cc
+++ b/src/librbd/mirror/snapshot/PromoteRequest.cc
@@ -27,6 +27,7 @@ namespace snapshot {
 
 using librbd::util::create_async_context_callback;
 using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
 
 template <typename I>
 void PromoteRequest<I>::send() {
@@ -312,6 +313,40 @@ void PromoteRequest<I>::handle_create_promote_snapshot(int r) {
     return;
   }
 
+  disable_non_primary_feature();
+}
+
+template <typename I>
+void PromoteRequest<I>::disable_non_primary_feature() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << dendl;
+
+  // remove the non-primary feature flag so that the image can be
+  // R/W by standard RBD clients
+  librados::ObjectWriteOperation op;
+  cls_client::set_features(&op, 0U, RBD_FEATURE_NON_PRIMARY);
+
+  auto aio_comp = create_rados_callback<
+    PromoteRequest<I>,
+    &PromoteRequest<I>::handle_disable_non_primary_feature>(this);
+  int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp,
+                                          &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void PromoteRequest<I>::handle_disable_non_primary_feature(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to disable non-primary feature: "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
   release_exclusive_lock();
 }
 

diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h
index 1583551f1d53483100c9604f14e3a7089aaa0903..f24138ff338633f8e12b2451ad1a060575213c22 100644 (file)
--- a/src/librbd/mirror/snapshot/PromoteRequest.h
+++ b/src/librbd/mirror/snapshot/PromoteRequest.h
@@ -66,6 +66,9 @@ private:
    * CREATE_PROMOTE_SNAPSHOT <--------------------/
    *    |
    *    v
+   * DISABLE_NON_PRIMARY_FEATURE
+   *    |
+   *    v
    * RELEASE_EXCLUSIVE_LOCK (skip if not needed)
    *    |
    *    v
@@ -128,6 +131,9 @@ private:
   void create_promote_snapshot();
   void handle_create_promote_snapshot(int r);
 
+  void disable_non_primary_feature();
+  void handle_disable_non_primary_feature(int r);
+
   void release_exclusive_lock();
   void handle_release_exclusive_lock(int r);