]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
librbd: wait for journal commit op event to be safely recorded
authorJason Dillaman <dillaman@redhat.com>
Tue, 19 Jul 2016 04:42:16 +0000 (00:42 -0400)
committerJason Dillaman <dillaman@redhat.com>
Wed, 17 Aug 2016 17:22:05 +0000 (13:22 -0400)
Operation request op finish events should not be fire and forget.
Instead, ensure the event is committed to the journal before
completing the op. This will avoid several possible split-brain
events during mirroring.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 47e0fbf231e52d00069c97b72c57c3158445bcf0)

Conflicts:
src/test/librbd/operation/test_mock_ResizeRequest.cc: no shrink restriction

12 files changed:
src/librbd/AsyncRequest.h
src/librbd/Journal.cc
src/librbd/Journal.h
src/librbd/operation/Request.cc
src/librbd/operation/Request.h
src/librbd/operation/ResizeRequest.cc
src/librbd/operation/SnapshotCreateRequest.cc
src/librbd/operation/SnapshotCreateRequest.h
src/librbd/operation/SnapshotRollbackRequest.cc
src/test/librbd/mock/MockJournal.h
src/test/librbd/operation/test_mock_ResizeRequest.cc
src/test/librbd/test_mock_fixture.cc

index 8ca84f534c8f7f138aac6b9a63855742c41d7874..3a1eb00531961d3a3cd6af6a978dfe2d05e898ca 100644 (file)
@@ -23,8 +23,7 @@ public:
   void complete(int r) {
     if (should_complete(r)) {
       r = filter_return_code(r);
-      finish(r);
-      delete this;
+      finish_and_destroy(r);
     }
   }
 
@@ -51,6 +50,12 @@ protected:
     return r;
   }
 
+  // NOTE: temporary until converted to new state machine format
+  virtual void finish_and_destroy(int r) {
+    finish(r);
+    delete this;
+  }
+
   virtual void finish(int r) {
     finish_request();
     m_on_finish->complete(r);
index c8c5c356d4309862244cbb2ed5709d00447fac3a..cff29bd7783e706ee5aa56df62f8e9b281c203af 100644 (file)
@@ -1006,7 +1006,7 @@ void Journal<I>::append_op_event(uint64_t op_tid,
 }
 
 template <typename I>
-void Journal<I>::commit_op_event(uint64_t op_tid, int r) {
+void Journal<I>::commit_op_event(uint64_t op_tid, int r, Context *on_safe) {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 10) << this << " " << __func__ << ": op_tid=" << op_tid << ", "
                  << "r=" << r << dendl;
@@ -1033,7 +1033,7 @@ void Journal<I>::commit_op_event(uint64_t op_tid, int r) {
 
   op_finish_future.flush(create_async_context_callback(
     m_image_ctx, new C_OpEventSafe(this, op_tid, op_start_future,
-                                   op_finish_future)));
+                                   op_finish_future, on_safe)));
 }
 
 template <typename I>
@@ -1645,7 +1645,8 @@ void Journal<I>::handle_io_event_safe(int r, uint64_t tid) {
 template <typename I>
 void Journal<I>::handle_op_event_safe(int r, uint64_t tid,
                                       const Future &op_start_future,
-                                      const Future &op_finish_future) {
+                                      const Future &op_finish_future,
+                                      Context *on_safe) {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 20) << this << " " << __func__ << ": r=" << r << ", "
                  << "tid=" << tid << dendl;
@@ -1661,7 +1662,7 @@ void Journal<I>::handle_op_event_safe(int r, uint64_t tid,
   m_journaler->committed(op_finish_future);
 
   // reduce the replay window after committing an op event
-  m_journaler->flush_commit_position(nullptr);
+  m_journaler->flush_commit_position(on_safe);
 }
 
 template <typename I>
index 7f085dfc6c4ab3314267a04d032d983996cb994c..76671ff8ba03605a2104ac958e13a634a83ab358 100644 (file)
@@ -143,7 +143,7 @@ public:
 
   void append_op_event(uint64_t op_tid, journal::EventEntry &&event_entry,
                        Context *on_safe);
-  void commit_op_event(uint64_t tid, int r);
+  void commit_op_event(uint64_t tid, int r, Context *on_safe);
   void replay_op_ready(uint64_t op_tid, Context *on_resume);
 
   void flush_event(uint64_t tid, Context *on_safe);
@@ -221,15 +221,17 @@ private:
     uint64_t tid;
     Future op_start_future;
     Future op_finish_future;
+    Context *on_safe;
 
     C_OpEventSafe(Journal *journal, uint64_t tid, const Future &op_start_future,
-                  const Future &op_finish_future)
+                  const Future &op_finish_future, Context *on_safe)
       : journal(journal), tid(tid), op_start_future(op_start_future),
-        op_finish_future(op_finish_future) {
+        op_finish_future(op_finish_future), on_safe(on_safe) {
     }
 
     virtual void finish(int r) {
-      journal->handle_op_event_safe(r, tid, op_start_future, op_finish_future);
+      journal->handle_op_event_safe(r, tid, op_start_future, op_finish_future,
+                                    on_safe);
     }
   };
 
@@ -348,7 +350,7 @@ private:
 
   void handle_io_event_safe(int r, uint64_t tid);
   void handle_op_event_safe(int r, uint64_t tid, const Future &op_start_future,
-                            const Future &op_finish_future);
+                            const Future &op_finish_future, Context *on_safe);
 
   void stop_recording();
 
index 216da1a983aef208262c70ab74f0350fc23cd480..32391a9c5a3a55c7fca4daa0afd776b74cc8fbce 100644 (file)
@@ -34,13 +34,40 @@ void Request<I>::send() {
 }
 
 template <typename I>
-void Request<I>::finish(int r) {
-  // automatically commit the event if we don't need to worry
-  // about affecting concurrent IO ops
-  if (r < 0 || !can_affect_io()) {
-    commit_op_event(r);
+Context *Request<I>::create_context_finisher(int r) {
+  // automatically commit the event if required (delete after commit)
+  if (m_appended_op_event && !m_committed_op_event &&
+      commit_op_event(r)) {
+    return nullptr;
   }
 
+  I &image_ctx = this->m_image_ctx;
+  CephContext *cct = image_ctx.cct;
+  ldout(cct, 10) << this << " " << __func__ << dendl;
+  return util::create_context_callback<Request<I>, &Request<I>::finish>(this);
+}
+
+template <typename I>
+void Request<I>::finish_and_destroy(int r) {
+  I &image_ctx = this->m_image_ctx;
+  CephContext *cct = image_ctx.cct;
+  ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
+
+  // automatically commit the event if required (delete after commit)
+  if (m_appended_op_event && !m_committed_op_event &&
+      commit_op_event(r)) {
+    return;
+  }
+
+  AsyncRequest<I>::finish_and_destroy(r);
+}
+
+template <typename I>
+void Request<I>::finish(int r) {
+  I &image_ctx = this->m_image_ctx;
+  CephContext *cct = image_ctx.cct;
+  ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
+
   assert(!m_appended_op_event || m_committed_op_event);
   AsyncRequest<I>::finish(r);
 }
@@ -61,12 +88,12 @@ bool Request<I>::append_op_event() {
 }
 
 template <typename I>
-void Request<I>::commit_op_event(int r) {
+bool Request<I>::commit_op_event(int r) {
   I &image_ctx = this->m_image_ctx;
   RWLock::RLocker snap_locker(image_ctx.snap_lock);
 
   if (!m_appended_op_event) {
-    return;
+    return false;
   }
 
   assert(m_op_tid != 0);
@@ -80,8 +107,27 @@ void Request<I>::commit_op_event(int r) {
 
     // ops will be canceled / completed before closing journal
     assert(image_ctx.journal->is_journal_ready());
-    image_ctx.journal->commit_op_event(m_op_tid, r);
+    image_ctx.journal->commit_op_event(m_op_tid, r,
+                                       new C_CommitOpEvent(this, r));
+    return true;
+  }
+  return false;
+}
+
+template <typename I>
+void Request<I>::handle_commit_op_event(int r, int original_ret_val) {
+  I &image_ctx = this->m_image_ctx;
+  CephContext *cct = image_ctx.cct;
+  ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to commit op event to journal: " << cpp_strerror(r)
+               << dendl;
+  }
+  if (original_ret_val < 0) {
+    r = original_ret_val;
   }
+  finish(r);
 }
 
 template <typename I>
@@ -108,7 +154,7 @@ void Request<I>::append_op_event(Context *on_safe) {
   m_op_tid = image_ctx.journal->allocate_op_tid();
   image_ctx.journal->append_op_event(
     m_op_tid, journal::EventEntry{create_event(m_op_tid)},
-    new C_OpEventSafe(this, on_safe));
+    new C_AppendOpEvent(this, on_safe));
 }
 
 template <typename I>
index be4d174be5d710153ef32e48232f927d49cd4587..77a8712e5ebf146e4de8fe66f99755aff5548669 100644 (file)
@@ -54,19 +54,16 @@ protected:
   }
 
   bool append_op_event();
-  void commit_op_event(int r);
 
   // NOTE: temporary until converted to new state machine format
-  Context *create_context_finisher() {
-    return util::create_context_callback<
-      Request<ImageCtxT>, &Request<ImageCtxT>::finish>(this);
-  }
+  Context *create_context_finisher(int r);
+  virtual void finish_and_destroy(int r) override;
 
 private:
-  struct C_OpEventSafe : public Context {
+  struct C_AppendOpEvent : public Context {
     Request *request;
     Context *on_safe;
-    C_OpEventSafe(Request *request, Context *on_safe)
+    C_AppendOpEvent(Request *request, Context *on_safe)
       : request(request), on_safe(on_safe) {
     }
     virtual void finish(int r) override {
@@ -77,6 +74,18 @@ private:
     }
   };
 
+  struct C_CommitOpEvent : public Context {
+    Request *request;
+    int ret_val;
+    C_CommitOpEvent(Request *request, int ret_val)
+      : request(request), ret_val(ret_val) {
+    }
+    virtual void finish(int r) override {
+      request->handle_commit_op_event(r, ret_val);
+      delete request;
+    }
+  };
+
   uint64_t m_op_tid = 0;
   bool m_appended_op_event = false;
   bool m_committed_op_event = false;
@@ -85,6 +94,9 @@ private:
   void append_op_event(Context *on_safe);
   void handle_op_event_safe(int r);
 
+  bool commit_op_event(int r);
+  void handle_commit_op_event(int r, int original_ret_val);
+
 };
 
 } // namespace operation
index a2ee7b0ebe5e4ae6e09daf5534c71c0daaed1724..a0687eb87d2a94fe5a18bfaa37216efc688fc54a 100644 (file)
@@ -106,7 +106,7 @@ Context *ResizeRequest<I>::handle_pre_block_writes(int *result) {
   if (*result < 0) {
     lderr(cct) << "failed to block writes: " << cpp_strerror(*result) << dendl;
     image_ctx.aio_work_queue->unblock_writes();
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   return send_append_op_event();
@@ -135,7 +135,7 @@ Context *ResizeRequest<I>::handle_append_op_event(int *result) {
     lderr(cct) << "failed to commit journal entry: " << cpp_strerror(*result)
                << dendl;
     image_ctx.aio_work_queue->unblock_writes();
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   return send_grow_object_map();
@@ -163,10 +163,10 @@ Context *ResizeRequest<I>::handle_trim_image(int *result) {
 
   if (*result == -ERESTART) {
     ldout(cct, 5) << "resize operation interrupted" << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   } else if (*result < 0) {
     lderr(cct) << "failed to trim image: " << cpp_strerror(*result) << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_invalidate_cache();
@@ -196,7 +196,7 @@ Context *ResizeRequest<I>::handle_invalidate_cache(int *result) {
   if (*result < 0) {
     lderr(cct) << "failed to invalidate cache: " << cpp_strerror(*result)
                << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_post_block_writes();
@@ -214,10 +214,7 @@ Context *ResizeRequest<I>::send_grow_object_map() {
   image_ctx.aio_work_queue->unblock_writes();
 
   if (m_original_size == m_new_size) {
-    if (!m_disable_journal) {
-      this->commit_op_event(0);
-    }
-    return this->create_context_finisher();
+    return this->create_context_finisher(0);
   } else if (m_new_size < m_original_size) {
     send_trim_image();
     return nullptr;
@@ -270,7 +267,7 @@ Context *ResizeRequest<I>::send_shrink_object_map() {
     image_ctx.owner_lock.put_read();
 
     update_size_and_overlap();
-    return this->create_context_finisher();
+    return this->create_context_finisher(0);
   }
 
   CephContext *cct = image_ctx.cct;
@@ -298,7 +295,7 @@ Context *ResizeRequest<I>::handle_shrink_object_map(int *result) {
 
   update_size_and_overlap();
   assert(*result == 0);
-  return this->create_context_finisher();
+  return this->create_context_finisher(0);
 }
 
 template <typename I>
@@ -322,7 +319,7 @@ Context *ResizeRequest<I>::handle_post_block_writes(int *result) {
     image_ctx.aio_work_queue->unblock_writes();
     lderr(cct) << "failed to block writes prior to header update: "
                << cpp_strerror(*result) << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_update_header();
@@ -374,12 +371,9 @@ Context *ResizeRequest<I>::handle_update_header(int *result) {
     lderr(cct) << "failed to update image header: " << cpp_strerror(*result)
                << dendl;
     image_ctx.aio_work_queue->unblock_writes();
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
-  if (!m_disable_journal) {
-    this->commit_op_event(0);
-  }
   return send_shrink_object_map();
 }
 
index cb92c6cac7646b0c53f95079eea61a9ebb51a087..c3dde26fc4e89520a24efa10756de73997430af5 100644 (file)
@@ -116,7 +116,7 @@ Context *SnapshotCreateRequest<I>::handle_suspend_aio(int *result) {
   if (*result < 0) {
     lderr(cct) << "failed to block writes: " << cpp_strerror(*result) << dendl;
     image_ctx.aio_work_queue->unblock_writes();
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_append_op_event();
@@ -147,7 +147,7 @@ Context *SnapshotCreateRequest<I>::handle_append_op_event(int *result) {
     image_ctx.aio_work_queue->unblock_writes();
     lderr(cct) << "failed to commit journal entry: " << cpp_strerror(*result)
                << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_allocate_snap_id();
@@ -176,10 +176,10 @@ Context *SnapshotCreateRequest<I>::handle_allocate_snap_id(int *result) {
 
   if (*result < 0) {
     save_result(result);
-    finalize(*result);
+    image_ctx.aio_work_queue->unblock_writes();
     lderr(cct) << "failed to allocate snapshot id: " << cpp_strerror(*result)
                << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_create_snap();
@@ -251,8 +251,8 @@ Context *SnapshotCreateRequest<I>::send_create_object_map() {
   if (image_ctx.object_map == nullptr || m_skip_object_map) {
     image_ctx.snap_lock.put_read();
 
-    finalize(0);
-    return this->create_context_finisher();
+    image_ctx.aio_work_queue->unblock_writes();
+    return this->create_context_finisher(0);
   }
 
   CephContext *cct = image_ctx.cct;
@@ -277,8 +277,8 @@ Context *SnapshotCreateRequest<I>::handle_create_object_map(int *result) {
 
   assert(*result == 0);
 
-  finalize(0);
-  return this->create_context_finisher();
+  image_ctx.aio_work_queue->unblock_writes();
+  return this->create_context_finisher(0);
 }
 
 template <typename I>
@@ -305,20 +305,8 @@ Context *SnapshotCreateRequest<I>::handle_release_snap_id(int *result) {
   assert(m_ret_val < 0);
   *result = m_ret_val;
 
-  finalize(m_ret_val);
-  return this->create_context_finisher();
-}
-
-template <typename I>
-void SnapshotCreateRequest<I>::finalize(int r) {
-  I &image_ctx = this->m_image_ctx;
-  CephContext *cct = image_ctx.cct;
-  ldout(cct, 5) << this << " " << __func__ << ": r=" << r << dendl;
-
-  if (r == 0) {
-    this->commit_op_event(0);
-  }
   image_ctx.aio_work_queue->unblock_writes();
+  return this->create_context_finisher(m_ret_val);
 }
 
 template <typename I>
index 35f8b53d60abc2894cd6c2fc26079b8349c473e5..62256bb30ef387f1b8c68d35f6c8586ef3097c3b 100644 (file)
@@ -106,7 +106,6 @@ private:
   void send_release_snap_id();
   Context *handle_release_snap_id(int *result);
 
-  void finalize(int r);
   void update_snap_context();
 
   void save_result(int *result) {
index 6dcf3a71083506b29f1eeb2c8d403c6d5bc9b187..3335b36683fe51e6420d373cf3f2136058f3b039 100644 (file)
@@ -107,7 +107,7 @@ Context *SnapshotRollbackRequest<I>::handle_block_writes(int *result) {
 
   if (*result < 0) {
     lderr(cct) << "failed to block writes: " << cpp_strerror(*result) << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_resize_image();
@@ -150,7 +150,7 @@ Context *SnapshotRollbackRequest<I>::handle_resize_image(int *result) {
   if (*result < 0) {
     lderr(cct) << "failed to resize image for rollback: "
                << cpp_strerror(*result) << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   send_rollback_object_map();
@@ -224,11 +224,11 @@ Context *SnapshotRollbackRequest<I>::handle_rollback_objects(int *result) {
 
   if (*result == -ERESTART) {
     ldout(cct, 5) << "snapshot rollback operation interrupted" << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   } else if (*result < 0) {
     lderr(cct) << "failed to rollback objects: " << cpp_strerror(*result)
                << dendl;
-    return this->create_context_finisher();
+    return this->create_context_finisher(*result);
   }
 
   return send_refresh_object_map();
@@ -276,7 +276,7 @@ Context *SnapshotRollbackRequest<I>::send_invalidate_cache() {
 
   apply();
   if (image_ctx.object_cacher == NULL) {
-    return this->create_context_finisher();
+    return this->create_context_finisher(0);
   }
 
   CephContext *cct = image_ctx.cct;
@@ -300,7 +300,7 @@ Context *SnapshotRollbackRequest<I>::handle_invalidate_cache(int *result) {
     lderr(cct) << "failed to invalidate cache: " << cpp_strerror(*result)
                << dendl;
   }
-  return this->create_context_finisher();
+  return this->create_context_finisher(*result);
 }
 
 template <typename I>
index f8ef75ad86419e482850729f7ad0d30adec7ba80..cfcb12c06ec15aa8f996566d0071aecdeb344e27 100644 (file)
@@ -55,7 +55,7 @@ struct MockJournal {
     append_op_event_mock(op_tid, event_entry, on_safe);
   }
 
-  MOCK_METHOD2(commit_op_event, void(uint64_t, int));
+  MOCK_METHOD3(commit_op_event, void(uint64_t, int, Context *));
   MOCK_METHOD2(replay_op_ready, void(uint64_t, Context *));
 
   MOCK_METHOD2(add_listener, void(journal::ListenerType,
index 34b0debde8b29ab0b4b8380e858232ba2190ae42..d5c1486b9c035cc82d42d8f436cafd72d5d7b077 100644 (file)
@@ -180,8 +180,8 @@ TEST_F(TestMockOperationResizeRequest, GrowSuccess) {
   expect_grow_object_map(mock_image_ctx);
   expect_block_writes(mock_image_ctx, 0);
   expect_update_header(mock_image_ctx, 0);
-  expect_commit_op_event(mock_image_ctx, 0);
   expect_unblock_writes(mock_image_ctx);
+  expect_commit_op_event(mock_image_ctx, 0);
   ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size * 2, 0, false));
 }
 
@@ -206,12 +206,29 @@ TEST_F(TestMockOperationResizeRequest, ShrinkSuccess) {
   expect_invalidate_cache(mock_image_ctx, 0);
   expect_block_writes(mock_image_ctx, 0);
   expect_update_header(mock_image_ctx, 0);
-  expect_commit_op_event(mock_image_ctx, 0);
   expect_shrink_object_map(mock_image_ctx);
   expect_unblock_writes(mock_image_ctx);
+  expect_commit_op_event(mock_image_ctx, 0);
   ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size / 2, 0, false));
 }
 
+TEST_F(TestMockOperationResizeRequest, ShrinkError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockExclusiveLock mock_exclusive_lock;
+  MockJournal mock_journal;
+  MockObjectMap mock_object_map;
+  initialize_features(ictx, mock_image_ctx, mock_exclusive_lock, mock_journal,
+                      mock_object_map);
+
+  InSequence seq;
+  expect_block_writes(mock_image_ctx, -EINVAL);
+  expect_unblock_writes(mock_image_ctx);
+  ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2, 0, false));
+}
+
 TEST_F(TestMockOperationResizeRequest, PreBlockWritesError) {
   librbd::ImageCtx *ictx;
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
index 4cc940cad47d306ce073a0caf9ccc92465d22a54..c2644eb534773d660da8786ef174f27fbc379f0c 100644 (file)
@@ -112,7 +112,8 @@ void TestMockFixture::expect_commit_op_event(librbd::MockImageCtx &mock_image_ct
   if (mock_image_ctx.journal != nullptr) {
     expect_is_journal_replaying(*mock_image_ctx.journal);
     expect_is_journal_ready(*mock_image_ctx.journal);
-    EXPECT_CALL(*mock_image_ctx.journal, commit_op_event(1U, r));
+    EXPECT_CALL(*mock_image_ctx.journal, commit_op_event(1U, r, _))
+                  .WillOnce(WithArg<2>(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)));
   }
 }