librbd: add no-op event when promoting an image

The rbd-mirror process needs an event in the journal
to properly detect the transition between primary and
non-primary state between peers.

Fixes: http://tracker.ceph.com/issues/19858
Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
index b1e4df33c8065df310955c033e2161ff07b6676c..f49359412059b67bb6f14e3c83552c61fdcfa79f 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror.sh
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -127,6 +127,25 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 compare_images ${POOL} ${image}
 
+# failover (unmodified)
+demote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+promote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+
+# failback (unmodified)
+demote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+promote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+compare_images ${POOL} ${image}
+
 # failover
 demote_image ${CLUSTER2} ${POOL} ${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}

diff --git a/src/librbd/journal/DemoteRequest.cc b/src/librbd/journal/DemoteRequest.cc
index 0fd3e335c442bc7fc8603b72f0f1955f0fb3a0aa..d51f777efe94524561d7baa7f9dfd6eb1bb80f63 100644 (file)
--- a/src/librbd/journal/DemoteRequest.cc
+++ b/src/librbd/journal/DemoteRequest.cc
@@ -131,7 +131,7 @@ void DemoteRequest<I>::append_event() {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 20) << dendl;
 
-  EventEntry event_entry{DemoteEvent{}, {}};
+  EventEntry event_entry{DemotePromoteEvent{}, {}};
   bufferlist event_entry_bl;
   ::encode(event_entry, event_entry_bl);
 

diff --git a/src/librbd/journal/PromoteRequest.cc b/src/librbd/journal/PromoteRequest.cc
index b541e48197eb3ddd331bb18c2d4d3cdcc9d7363c..fb85bcb86618cd7acdc7da0384c20d191f63cd0e 100644 (file)
--- a/src/librbd/journal/PromoteRequest.cc
+++ b/src/librbd/journal/PromoteRequest.cc
@@ -102,6 +102,93 @@ void PromoteRequest<I>::handle_allocate_tag(int r) {
   if (r < 0) {
     m_ret_val = r;
     lderr(cct) << "failed to allocate tag: " << cpp_strerror(r) << dendl;
+    shut_down();
+    return;
+  }
+
+  m_tag_tid = m_tag.tid;
+  append_event();
+}
+
+template <typename I>
+void PromoteRequest<I>::append_event() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  EventEntry event_entry{DemotePromoteEvent{}, {}};
+  bufferlist event_entry_bl;
+  ::encode(event_entry, event_entry_bl);
+
+  m_journaler->start_append(0, 0, 0);
+  m_future = m_journaler->append(m_tag_tid, event_entry_bl);
+
+  auto ctx = create_context_callback<
+    PromoteRequest<I>, &PromoteRequest<I>::handle_append_event>(this);
+  m_future.flush(ctx);
+}
+
+template <typename I>
+void PromoteRequest<I>::handle_append_event(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << "r=" << r << dendl;
+
+  if (r < 0) {
+    m_ret_val = r;
+    lderr(cct) << "failed to append promotion journal event: "
+               << cpp_strerror(r) << dendl;
+    stop_append();
+    return;
+  }
+
+  commit_event();
+}
+
+template <typename I>
+void PromoteRequest<I>::commit_event() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  m_journaler->committed(m_future);
+
+  auto ctx = create_context_callback<
+    PromoteRequest<I>, &PromoteRequest<I>::handle_commit_event>(this);
+  m_journaler->flush_commit_position(ctx);
+}
+
+template <typename I>
+void PromoteRequest<I>::handle_commit_event(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << "r=" << r << dendl;
+
+  if (r < 0) {
+    m_ret_val = r;
+    lderr(cct) << "failed to flush promote commit position: "
+               << cpp_strerror(r) << dendl;
+  }
+
+  stop_append();
+}
+
+template <typename I>
+void PromoteRequest<I>::stop_append() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  auto ctx = create_context_callback<
+    PromoteRequest<I>, &PromoteRequest<I>::handle_stop_append>(this);
+  m_journaler->stop_append(ctx);
+}
+
+template <typename I>
+void PromoteRequest<I>::handle_stop_append(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << "r=" << r << dendl;
+
+  if (r < 0) {
+    if (m_ret_val == 0) {
+      m_ret_val = r;
+    }
+    lderr(cct) << "failed to stop journal append: " << cpp_strerror(r) << dendl;
   }
 
   shut_down();

diff --git a/src/librbd/journal/PromoteRequest.h b/src/librbd/journal/PromoteRequest.h
index 4ef1d116c5833735a4d6fcbc256fdf7fec0c4e2c..0d01f596108d87502bfc9794b23d77bd1cfe2d3e 100644 (file)
--- a/src/librbd/journal/PromoteRequest.h
+++ b/src/librbd/journal/PromoteRequest.h
@@ -7,6 +7,7 @@
 #include "include/int_types.h"
 #include "common/Mutex.h"
 #include "cls/journal/cls_journal_types.h"
+#include "journal/Future.h"
 #include "librbd/journal/Types.h"
 #include "librbd/journal/TypeTraits.h"
 
@@ -37,13 +38,22 @@ private:
    * <start>
    *    |
    *    v
-   *  OPEN
-   *    |
-   *    v
-   * ALLOCATE_TAG
-   *    |
-   *    v
-   * SHUT_DOWN
+   * OPEN * * * * * * * * * *
+   *    |                   *
+   *    v                   *
+   * ALLOCATE_TAG * * * * * *
+   *    |                   *
+   *    v                   *
+   * APPEND_EVENT * * *     *
+   *    |             *     *
+   *    v             *     *
+   * COMMIT_EVENT     *     *
+   *    |             *     *
+   *    v             *     *
+   * STOP_APPEND <* * *     *
+   *    |                   *
+   *    v                   *
+   * SHUT_DOWN <* * * * * * *
    *    |
    *    v
    * <finish>
@@ -52,6 +62,7 @@ private:
    */
 
   typedef typename TypeTraits<ImageCtxT>::Journaler Journaler;
+  typedef typename TypeTraits<ImageCtxT>::Future Future;
 
   ImageCtxT *m_image_ctx;
   bool m_force;
@@ -66,6 +77,7 @@ private:
   TagData m_tag_data;
 
   cls::journal::Tag m_tag;
+  Future m_future;
 
   void send_open();
   void handle_open(int r);
@@ -73,6 +85,15 @@ private:
   void allocate_tag();
   void handle_allocate_tag(int r);
 
+  void append_event();
+  void handle_append_event(int r);
+
+  void commit_event();
+  void handle_commit_event(int r);
+
+  void stop_append();
+  void handle_stop_append(int r);
+
   void shut_down();
   void handle_shut_down(int r);
 

diff --git a/src/librbd/journal/Replay.cc b/src/librbd/journal/Replay.cc
index 481f7510c53e11d577306b300fdd970e87b6ce8f..624b213b8f6624303f1cf3a63a03b84c94da1657 100644 (file)
--- a/src/librbd/journal/Replay.cc
+++ b/src/librbd/journal/Replay.cc
@@ -670,10 +670,10 @@ void Replay<I>::handle_event(const journal::FlattenEvent &event,
 }
 
 template <typename I>
-void Replay<I>::handle_event(const journal::DemoteEvent &event,
+void Replay<I>::handle_event(const journal::DemotePromoteEvent &event,
                             Context *on_ready, Context *on_safe) {
   CephContext *cct = m_image_ctx.cct;
-  ldout(cct, 20) << ": Demote event" << dendl;
+  ldout(cct, 20) << ": Demote/Promote event" << dendl;
   on_ready->complete(0);
   on_safe->complete(0);
 }

diff --git a/src/librbd/journal/Replay.h b/src/librbd/journal/Replay.h
index afe71b994f7ba2c11c69043d9b2375dd2bbf8caa..d4a65e089bb2dbc25cec7fbb288370b7d91b8a52 100644 (file)
--- a/src/librbd/journal/Replay.h
+++ b/src/librbd/journal/Replay.h
@@ -157,7 +157,7 @@ private:
                     Context *on_safe);
   void handle_event(const FlattenEvent &event, Context *on_ready,
                     Context *on_safe);
-  void handle_event(const DemoteEvent &event, Context *on_ready,
+  void handle_event(const DemotePromoteEvent &event, Context *on_ready,
                     Context *on_safe);
   void handle_event(const SnapLimitEvent &event, Context *on_ready,
                     Context *on_safe);

diff --git a/src/librbd/journal/Types.cc b/src/librbd/journal/Types.cc
index c30a5752df290b8044c76edaa75d090accb8c827..857542bd74b11b8878929fe12c02421e780af4ca 100644 (file)
--- a/src/librbd/journal/Types.cc
+++ b/src/librbd/journal/Types.cc
@@ -272,13 +272,13 @@ void ResizeEvent::dump(Formatter *f) const {
   f->dump_unsigned("size", size);
 }
 
-void DemoteEvent::encode(bufferlist& bl) const {
+void DemotePromoteEvent::encode(bufferlist& bl) const {
 }
 
-void DemoteEvent::decode(__u8 version, bufferlist::iterator& it) {
+void DemotePromoteEvent::decode(__u8 version, bufferlist::iterator& it) {
 }
 
-void DemoteEvent::dump(Formatter *f) const {
+void DemotePromoteEvent::dump(Formatter *f) const {
 }
 
 void UpdateFeaturesEvent::encode(bufferlist& bl) const {
@@ -400,8 +400,8 @@ void EventEntry::decode(bufferlist::iterator& it) {
   case EVENT_TYPE_FLATTEN:
     event = FlattenEvent();
     break;
-  case EVENT_TYPE_DEMOTE:
-    event = DemoteEvent();
+  case EVENT_TYPE_DEMOTE_PROMOTE:
+    event = DemotePromoteEvent();
     break;
   case EVENT_TYPE_UPDATE_FEATURES:
     event = UpdateFeaturesEvent();
@@ -484,7 +484,7 @@ void EventEntry::generate_test_instances(std::list<EventEntry *> &o) {
 
   o.push_back(new EventEntry(FlattenEvent(123), utime_t(1, 1)));
 
-  o.push_back(new EventEntry(DemoteEvent()));
+  o.push_back(new EventEntry(DemotePromoteEvent()));
 
   o.push_back(new EventEntry(UpdateFeaturesEvent()));
   o.push_back(new EventEntry(UpdateFeaturesEvent(123, 127, true), utime_t(1, 1)));
@@ -749,8 +749,8 @@ std::ostream &operator<<(std::ostream &out, const EventType &type) {
   case EVENT_TYPE_FLATTEN:
     out << "Flatten";
     break;
-  case EVENT_TYPE_DEMOTE:
-    out << "Demote";
+  case EVENT_TYPE_DEMOTE_PROMOTE:
+    out << "Demote/Promote";
     break;
   case EVENT_TYPE_UPDATE_FEATURES:
     out << "UpdateFeatures";

diff --git a/src/librbd/journal/Types.h b/src/librbd/journal/Types.h
index 9c8057bd36a00bcae82fa846db5a6b05e55c4a9e..adc6f29557520d156848078e7750d254ede13ae4 100644 (file)
--- a/src/librbd/journal/Types.h
+++ b/src/librbd/journal/Types.h
@@ -37,7 +37,7 @@ enum EventType {
   EVENT_TYPE_RENAME          = 10,
   EVENT_TYPE_RESIZE          = 11,
   EVENT_TYPE_FLATTEN         = 12,
-  EVENT_TYPE_DEMOTE          = 13,
+  EVENT_TYPE_DEMOTE_PROMOTE  = 13,
   EVENT_TYPE_SNAP_LIMIT      = 14,
   EVENT_TYPE_UPDATE_FEATURES = 15,
   EVENT_TYPE_METADATA_SET    = 16,
@@ -316,8 +316,9 @@ struct FlattenEvent : public OpEventBase {
   using OpEventBase::dump;
 };
 
-struct DemoteEvent {
-  static const EventType TYPE = static_cast<EventType>(EVENT_TYPE_DEMOTE);
+struct DemotePromoteEvent {
+  static const EventType TYPE = static_cast<EventType>(
+    EVENT_TYPE_DEMOTE_PROMOTE);
 
   void encode(bufferlist& bl) const;
   void decode(__u8 version, bufferlist::iterator& it);
@@ -395,7 +396,7 @@ typedef boost::variant<AioDiscardEvent,
                        RenameEvent,
                        ResizeEvent,
                        FlattenEvent,
-                       DemoteEvent,
+                       DemotePromoteEvent,
                       SnapLimitEvent,
                        UpdateFeaturesEvent,
                        MetadataSetEvent,

diff --git a/src/test/librbd/journal/test_mock_PromoteRequest.cc b/src/test/librbd/journal/test_mock_PromoteRequest.cc
index 805e76aaed4ae403acbe702f03c0738853c8de1f..209249955b155a9ff18aa8be3c63b60afb22de75 100644 (file)
--- a/src/test/librbd/journal/test_mock_PromoteRequest.cc
+++ b/src/test/librbd/journal/test_mock_PromoteRequest.cc
@@ -24,6 +24,7 @@ namespace journal {
 template <>
 struct TypeTraits<MockTestImageCtx> {
   typedef ::journal::MockJournalerProxy Journaler;
+  typedef ::journal::MockFutureProxy  Future;
 };
 
 template <>
@@ -63,7 +64,9 @@ namespace librbd {
 namespace journal {
 
 using ::testing::_;
+using ::testing::A;
 using ::testing::InSequence;
+using ::testing::Return;
 using ::testing::WithArg;
 
 class TestMockJournalPromoteRequest : public TestMockFixture {
@@ -95,6 +98,35 @@ public:
       .WillOnce(WithArg<3>(CompleteContext(r, static_cast<ContextWQ*>(NULL))));
   }
 
+  void expect_append_journaler(::journal::MockJournaler &mock_journaler) {
+    EXPECT_CALL(mock_journaler, append(_, _))
+                  .WillOnce(Return(::journal::MockFutureProxy()));
+  }
+
+  void expect_future_flush(::journal::MockFuture &mock_future, int r) {
+    EXPECT_CALL(mock_future, flush(_))
+                  .WillOnce(CompleteContext(r, static_cast<ContextWQ*>(NULL)));
+  }
+
+  void expect_future_committed(::journal::MockJournaler &mock_journaler) {
+    EXPECT_CALL(mock_journaler, committed(A<const ::journal::MockFutureProxy &>()));
+  }
+
+  void expect_flush_commit_position(::journal::MockJournaler &mock_journaler,
+                                    int r) {
+    EXPECT_CALL(mock_journaler, flush_commit_position(_))
+                  .WillOnce(CompleteContext(r, static_cast<ContextWQ*>(NULL)));
+  }
+
+  void expect_start_append(::journal::MockJournaler &mock_journaler) {
+    EXPECT_CALL(mock_journaler, start_append(_, _, _));
+  }
+
+  void expect_stop_append(::journal::MockJournaler &mock_journaler, int r) {
+    EXPECT_CALL(mock_journaler, stop_append(_))
+                  .WillOnce(CompleteContext(r, static_cast<ContextWQ*>(NULL)));
+  }
+
   void expect_shut_down_journaler(::journal::MockJournaler &mock_journaler,
                                   int r) {
     EXPECT_CALL(mock_journaler, shut_down(_))
@@ -120,6 +152,15 @@ TEST_F(TestMockJournalPromoteRequest, SuccessOrderly) {
   expect_open_journaler(mock_image_ctx, mock_open_request, 0);
   expect_allocate_tag(mock_journaler,
                       {Journal<>::ORPHAN_MIRROR_UUID, true, 567, 1}, 0);
+
+  ::journal::MockFuture mock_future;
+  expect_start_append(mock_journaler);
+  expect_append_journaler(mock_journaler);
+  expect_future_flush(mock_future, 0);
+  expect_future_committed(mock_journaler);
+  expect_flush_commit_position(mock_journaler, 0);
+  expect_stop_append(mock_journaler, 0);
+
   expect_shut_down_journaler(mock_journaler, 0);
 
   C_SaferCond ctx;
@@ -145,6 +186,15 @@ TEST_F(TestMockJournalPromoteRequest, SuccessForced) {
   expect_open_journaler(mock_image_ctx, mock_open_request, 0);
   expect_allocate_tag(mock_journaler,
                       {Journal<>::LOCAL_MIRROR_UUID, true, 567, 0}, 0);
+
+  ::journal::MockFuture mock_future;
+  expect_start_append(mock_journaler);
+  expect_append_journaler(mock_journaler);
+  expect_future_flush(mock_future, 0);
+  expect_future_committed(mock_journaler);
+  expect_flush_commit_position(mock_journaler, 0);
+  expect_stop_append(mock_journaler, 0);
+
   expect_shut_down_journaler(mock_journaler, 0);
 
   C_SaferCond ctx;
@@ -201,6 +251,72 @@ TEST_F(TestMockJournalPromoteRequest, AllocateTagError) {
   ASSERT_EQ(-EBADMSG, ctx.wait());
 }
 
+TEST_F(TestMockJournalPromoteRequest, AppendEventError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  ::journal::MockJournaler mock_journaler;
+  MockOpenRequest mock_open_request;
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_construct_journaler(mock_journaler);
+  expect_open_journaler(mock_image_ctx, mock_open_request, 0);
+  expect_allocate_tag(mock_journaler,
+                      {Journal<>::ORPHAN_MIRROR_UUID, true, 567, 1}, 0);
+
+  ::journal::MockFuture mock_future;
+  expect_start_append(mock_journaler);
+  expect_append_journaler(mock_journaler);
+  expect_future_flush(mock_future, -EPERM);
+  expect_stop_append(mock_journaler, 0);
+
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  C_SaferCond ctx;
+  auto req = MockPromoteRequest::create(&mock_image_ctx, false, &ctx);
+  req->send();
+  ASSERT_EQ(-EPERM, ctx.wait());
+}
+
+TEST_F(TestMockJournalPromoteRequest, CommitEventError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  ::journal::MockJournaler mock_journaler;
+  MockOpenRequest mock_open_request;
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_construct_journaler(mock_journaler);
+  expect_open_journaler(mock_image_ctx, mock_open_request, 0);
+  expect_allocate_tag(mock_journaler,
+                      {Journal<>::ORPHAN_MIRROR_UUID, true, 567, 1}, 0);
+
+  ::journal::MockFuture mock_future;
+  expect_start_append(mock_journaler);
+  expect_append_journaler(mock_journaler);
+  expect_future_flush(mock_future, 0);
+  expect_future_committed(mock_journaler);
+  expect_flush_commit_position(mock_journaler, -EINVAL);
+  expect_stop_append(mock_journaler, 0);
+
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  C_SaferCond ctx;
+  auto req = MockPromoteRequest::create(&mock_image_ctx, false, &ctx);
+  req->send();
+  ASSERT_EQ(-EINVAL, ctx.wait());
+}
+
 TEST_F(TestMockJournalPromoteRequest, ShutDownError) {
   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 
@@ -218,6 +334,15 @@ TEST_F(TestMockJournalPromoteRequest, ShutDownError) {
   expect_open_journaler(mock_image_ctx, mock_open_request, 0);
   expect_allocate_tag(mock_journaler,
                       {Journal<>::LOCAL_MIRROR_UUID, true, 567, 0}, 0);
+
+  ::journal::MockFuture mock_future;
+  expect_start_append(mock_journaler);
+  expect_append_journaler(mock_journaler);
+  expect_future_flush(mock_future, 0);
+  expect_future_committed(mock_journaler);
+  expect_flush_commit_position(mock_journaler, 0);
+  expect_stop_append(mock_journaler, 0);
+
   expect_shut_down_journaler(mock_journaler, -EINVAL);
 
   C_SaferCond ctx;