librbd: image demotion should record new demote journal event

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index 1871774ffe058f76f540717e85b54ca36ff1332d..7b6f6b1a7aee9f8d4a89f0a9457abebced29afc6 100644 (file)
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -162,6 +162,7 @@ public:
 
 template <typename I, typename J>
 int open_journaler(I *image_ctx, J *journaler, bool *initialized,
+                   cls::journal::Client *client,
                    journal::ImageClientMeta *client_meta,
                    journal::TagData *tag_data) {
   C_SaferCond init_ctx;
@@ -172,14 +173,13 @@ int open_journaler(I *image_ctx, J *journaler, bool *initialized,
     return r;
   }
 
-  cls::journal::Client client;
-  r = journaler->get_cached_client(Journal<ImageCtx>::IMAGE_CLIENT_ID, &client);
+  r = journaler->get_cached_client(Journal<ImageCtx>::IMAGE_CLIENT_ID, client);
   if (r < 0) {
     return r;
   }
 
   librbd::journal::ClientData client_data;
-  bufferlist::iterator bl_it = client.data.begin();
+  bufferlist::iterator bl_it = client->data.begin();
   try {
     ::decode(client_data, bl_it);
   } catch (const buffer::error &err) {
@@ -207,6 +207,37 @@ int open_journaler(I *image_ctx, J *journaler, bool *initialized,
   return 0;
 }
 
+template <typename J>
+int allocate_journaler_tag(CephContext *cct, J *journaler,
+                           const cls::journal::Client &client,
+                           uint64_t tag_class,
+                           const journal::TagData &prev_tag_data,
+                           const std::string &mirror_uuid,
+                           cls::journal::Tag *new_tag) {
+  journal::TagData tag_data;
+  if (!client.commit_position.object_positions.empty()) {
+    auto position = client.commit_position.object_positions.front();
+    tag_data.predecessor_commit_valid = true;
+    tag_data.predecessor_tag_tid = position.tag_tid;
+    tag_data.predecessor_entry_tid = position.entry_tid;
+  }
+  tag_data.predecessor_mirror_uuid = prev_tag_data.mirror_uuid;
+  tag_data.mirror_uuid = mirror_uuid;
+
+  bufferlist tag_bl;
+  ::encode(tag_data, tag_bl);
+
+  C_SaferCond allocate_tag_ctx;
+  journaler->allocate_tag(tag_class, tag_bl, new_tag, &allocate_tag_ctx);
+
+  int r = allocate_tag_ctx.wait();
+  if (r < 0) {
+    lderr(cct) << "failed to allocate tag: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  return 0;
+}
+
 } // anonymous namespace
 
 using util::create_async_context_callback;
@@ -340,23 +371,14 @@ int Journal<I>::create(librados::IoCtx &io_ctx, const std::string &image_id,
     return r;
   }
 
-  // create tag class for this image's journal events
-  journal::TagData tag_data;
-  tag_data.mirror_uuid = (!non_primary ? LOCAL_MIRROR_UUID :
-                                         ORPHAN_MIRROR_UUID);
-
-  bufferlist tag_data_bl;
-  ::encode(tag_data, tag_data_bl);
-
-  C_SaferCond tag_ctx;
+  cls::journal::Client client;
   cls::journal::Tag tag;
-  journaler.allocate_tag(cls::journal::Tag::TAG_CLASS_NEW, tag_data_bl,
-                         &tag, &tag_ctx);
-  r = tag_ctx.wait();
-  if (r < 0) {
-    lderr(cct) << "failed to allocate journal tag: " << cpp_strerror(r)
-               << dendl;
-  }
+  journal::TagData tag_data;
+  std::string mirror_uuid = (!non_primary ? LOCAL_MIRROR_UUID :
+                                            ORPHAN_MIRROR_UUID);
+  r = allocate_journaler_tag(cct, &journaler, client,
+                             cls::journal::Tag::TAG_CLASS_NEW,
+                             tag_data, mirror_uuid, &tag);
 
   bufferlist client_data;
   ::encode(journal::ClientData{journal::ImageClientMeta{tag.tag_class}},
@@ -474,10 +496,11 @@ int Journal<I>::get_tag_owner(I *image_ctx, std::string *mirror_uuid) {
                       image_ctx->cct->_conf->rbd_journal_commit_age);
 
   bool initialized;
+  cls::journal::Client client;
   journal::ImageClientMeta client_meta;
   journal::TagData tag_data;
-  int r = open_journaler(image_ctx, &journaler, &initialized, &client_meta,
-                         &tag_data);
+  int r = open_journaler(image_ctx, &journaler, &initialized, &client,
+                         &client_meta, &tag_data);
   if (r >= 0) {
     *mirror_uuid = tag_data.mirror_uuid;
   }
@@ -489,18 +512,19 @@ int Journal<I>::get_tag_owner(I *image_ctx, std::string *mirror_uuid) {
 }
 
 template <typename I>
-int Journal<I>::allocate_tag(I *image_ctx, const std::string &mirror_uuid) {
+int Journal<I>::request_resync(I *image_ctx) {
   CephContext *cct = image_ctx->cct;
-  ldout(cct, 20) << __func__ << ": mirror_uuid=" << mirror_uuid << dendl;
+  ldout(cct, 20) << __func__ << dendl;
 
   Journaler journaler(image_ctx->md_ctx, image_ctx->id, IMAGE_CLIENT_ID,
                       image_ctx->cct->_conf->rbd_journal_commit_age);
 
   bool initialized;
+  cls::journal::Client client;
   journal::ImageClientMeta client_meta;
   journal::TagData tag_data;
-  int r = open_journaler(image_ctx, &journaler, &initialized, &client_meta,
-                         &tag_data);
+  int r = open_journaler(image_ctx, &journaler, &initialized, &client,
+                         &client_meta, &tag_data);
   BOOST_SCOPE_EXIT_ALL(&journaler, &initialized) {
     if (initialized) {
       journaler.shut_down();
@@ -511,41 +535,25 @@ int Journal<I>::allocate_tag(I *image_ctx, const std::string &mirror_uuid) {
     return r;
   }
 
-  cls::journal::Client client;
-  r = journaler.get_cached_client(IMAGE_CLIENT_ID, &client);
-  if (r < 0) {
-    lderr(cct) << "failed to retrieve client" << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  if (!client.commit_position.object_positions.empty()) {
-    auto position = client.commit_position.object_positions.front();
-    tag_data.predecessor_commit_valid = true;
-    tag_data.predecessor_tag_tid = position.tag_tid;
-    tag_data.predecessor_entry_tid = position.entry_tid;
-  }
-  tag_data.predecessor_mirror_uuid = tag_data.mirror_uuid;
-  tag_data.mirror_uuid = mirror_uuid;
+  client_meta.resync_requested = true;
 
-  bufferlist tag_bl;
-  ::encode(tag_data, tag_bl);
+  journal::ClientData client_data(client_meta);
+  bufferlist client_data_bl;
+  ::encode(client_data, client_data_bl);
 
-  C_SaferCond allocate_tag_ctx;
-  cls::journal::Tag tag;
-  journaler.allocate_tag(client_meta.tag_class, tag_bl, &tag,
-                         &allocate_tag_ctx);
+  C_SaferCond update_client_ctx;
+  journaler.update_client(client_data_bl, &update_client_ctx);
 
-  r = allocate_tag_ctx.wait();
+  r = update_client_ctx.wait();
   if (r < 0) {
-    lderr(cct) << "failed to allocate tag: " << cpp_strerror(r) << dendl;
+    lderr(cct) << "failed to update client: " << cpp_strerror(r) << dendl;
     return r;
   }
-
   return 0;
 }
 
 template <typename I>
-int Journal<I>::request_resync(I *image_ctx) {
+int Journal<I>::promote(I *image_ctx) {
   CephContext *cct = image_ctx->cct;
   ldout(cct, 20) << __func__ << dendl;
 
@@ -553,10 +561,11 @@ int Journal<I>::request_resync(I *image_ctx) {
                       image_ctx->cct->_conf->rbd_journal_commit_age);
 
   bool initialized;
+  cls::journal::Client client;
   journal::ImageClientMeta client_meta;
   journal::TagData tag_data;
-  int r = open_journaler(image_ctx, &journaler, &initialized, &client_meta,
-                         &tag_data);
+  int r = open_journaler(image_ctx, &journaler, &initialized, &client,
+                         &client_meta, &tag_data);
   BOOST_SCOPE_EXIT_ALL(&journaler, &initialized) {
     if (initialized) {
       journaler.shut_down();
@@ -567,20 +576,13 @@ int Journal<I>::request_resync(I *image_ctx) {
     return r;
   }
 
-  client_meta.resync_requested = true;
-
-  journal::ClientData client_data(client_meta);
-  bufferlist client_data_bl;
-  ::encode(client_data, client_data_bl);
-
-  C_SaferCond update_client_ctx;
-  journaler.update_client(client_data_bl, &update_client_ctx);
-
-  r = update_client_ctx.wait();
+  cls::journal::Tag new_tag;
+  r = allocate_journaler_tag(cct, &journaler, client, client_meta.tag_class,
+                             tag_data, LOCAL_MIRROR_UUID, &new_tag);
   if (r < 0) {
-    lderr(cct) << "failed to update client: " << cpp_strerror(r) << dendl;
     return r;
   }
+
   return 0;
 }
 
@@ -650,6 +652,65 @@ bool Journal<I>::is_tag_owner() const {
   return (m_tag_data.mirror_uuid == LOCAL_MIRROR_UUID);
 }
 
+template <typename I>
+int Journal<I>::demote() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 20) << __func__ << dendl;
+
+  Mutex::Locker locker(m_lock);
+  assert(m_journaler != nullptr && is_tag_owner());
+
+  cls::journal::Client client;
+  int r = m_journaler->get_cached_client(IMAGE_CLIENT_ID, &client);
+  if (r < 0) {
+    lderr(cct) << "failed to retrieve client: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  cls::journal::Tag new_tag;
+  r = allocate_journaler_tag(cct, m_journaler, client, m_tag_class,
+                             m_tag_data, ORPHAN_MIRROR_UUID, &new_tag);
+  if (r < 0) {
+    return r;
+  }
+
+  bufferlist::iterator tag_data_bl_it = new_tag.data.begin();
+  r = C_DecodeTag::decode(&tag_data_bl_it, &m_tag_data);
+  if (r < 0) {
+    lderr(cct) << "failed to decode newly allocated tag" << dendl;
+    return r;
+  }
+
+  journal::EventEntry event_entry{journal::DemoteEvent{}};
+  bufferlist event_entry_bl;
+  ::encode(event_entry, event_entry_bl);
+
+  m_tag_tid = new_tag.tid;
+  Future future = m_journaler->append(m_tag_tid, event_entry_bl);
+  C_SaferCond ctx;
+  future.flush(&ctx);
+
+  r = ctx.wait();
+  if (r < 0) {
+    lderr(cct) << "failed to append demotion journal event: " << cpp_strerror(r)
+               << dendl;
+    return r;
+  }
+
+  m_journaler->committed(future);
+  C_SaferCond flush_ctx;
+  m_journaler->flush_commit_position(&flush_ctx);
+
+  r = flush_ctx.wait();
+  if (r < 0) {
+    lderr(cct) << "failed to flush demotion commit position: "
+               << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
 template <typename I>
 void Journal<I>::allocate_local_tag(Context *on_finish) {
   CephContext *cct = m_image_ctx.cct;

diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h
index 5258ae7db3fa390cb0f242bc3ebf2eef4c2f3c55..e714b8c4d74f328f69816ac385afe68e70c26b98 100644 (file)
--- a/src/librbd/Journal.h
+++ b/src/librbd/Journal.h
@@ -101,8 +101,8 @@ public:
 
   static int is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner);
   static int get_tag_owner(ImageCtxT *image_ctx, std::string *mirror_uuid);
-  static int allocate_tag(ImageCtxT *image_ctx, const std::string &mirror_uuid);
   static int request_resync(ImageCtxT *image_ctx);
+  static int promote(ImageCtxT *image_ctx);
 
   bool is_journal_ready() const;
   bool is_journal_replaying() const;
@@ -113,6 +113,8 @@ public:
   void close(Context *on_finish);
 
   bool is_tag_owner() const;
+  int demote();
+
   void allocate_local_tag(Context *on_finish);
   void allocate_tag(const std::string &mirror_uuid,
                     const std::string &predecessor_mirror_uuid,

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 270076d59221d8cdf5b6340c54785592b48acf2d..8408de1ffe3c8a554e4cc8666950e4f087dbddc7 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -2686,7 +2686,7 @@ remove_mirroring_image:
     // TODO: need interlock with local rbd-mirror daemon to ensure it has stopped
     //       replay
 
-    r = Journal<>::allocate_tag(ictx, Journal<>::LOCAL_MIRROR_UUID);
+    r = Journal<>::promote(ictx);
     if (r < 0) {
       lderr(cct) << "failed to promote image: " << cpp_strerror(r)
                  << dendl;
@@ -2718,16 +2718,32 @@ remove_mirroring_image:
     }
 
     RWLock::RLocker owner_lock(ictx->owner_lock);
+    if (ictx->exclusive_lock == nullptr) {
+      lderr(cct) << "exclusive lock is not active" << dendl;
+      return -EINVAL;
+    }
+
     C_SaferCond lock_ctx;
     ictx->exclusive_lock->request_lock(&lock_ctx);
     r = lock_ctx.wait();
     if (r < 0) {
       lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl;
+      return r;
     } else if (!ictx->exclusive_lock->is_lock_owner()) {
       lderr(cct) << "failed to acquire exclusive lock" << dendl;
+      return -EROFS;
+    }
+
+    RWLock::RLocker snap_locker(ictx->snap_lock);
+    if (ictx->journal == nullptr) {
+      lderr(cct) << "journal is not active" << dendl;
+      return -EINVAL;
+    } else if (!ictx->journal->is_tag_owner()) {
+      lderr(cct) << "image is not currently the primary" << dendl;
+      return -EINVAL;
     }
 
-    r = Journal<>::allocate_tag(ictx, Journal<>::ORPHAN_MIRROR_UUID);
+    r = ictx->journal->demote();
     if (r < 0) {
       lderr(cct) << "failed to demote image: " << cpp_strerror(r)
                  << dendl;