librbd: basic policy for journal handling

This will allow rbd-mirror to override the allocation of a new
tag and to restart rbd-mirror's image replayer when a watch error
forces the loss of the exclusive lock.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index d4c257d0e760ccdcba7a880d92fce505e49078dc..770e87189c6fc5a6f13e3e7f224af19d9c28edc1 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -21,6 +21,7 @@
 #include "librbd/ImageState.h"
 #include "librbd/ImageWatcher.h"
 #include "librbd/Journal.h"
+#include "librbd/journal/StandardPolicy.h"
 #include "librbd/LibrbdAdminSocketHook.h"
 #include "librbd/ObjectMap.h"
 #include "librbd/Operations.h"
@@ -190,6 +191,7 @@ struct C_InvalidateCache : public Context {
                                   thread_pool_singleton);
 
     exclusive_lock_policy = new exclusive_lock::StandardPolicy(this);
+    journal_policy = new journal::StandardPolicy(this);
   }
 
   ImageCtx::~ImageCtx() {
@@ -221,6 +223,7 @@ struct C_InvalidateCache : public Context {
     op_work_queue->drain();
     aio_work_queue->drain();
 
+    delete journal_policy;
     delete exclusive_lock_policy;
     delete op_work_queue;
     delete aio_work_queue;
@@ -1063,4 +1066,17 @@ struct C_InvalidateCache : public Context {
     delete exclusive_lock_policy;
     exclusive_lock_policy = policy;
   }
+
+  journal::Policy *ImageCtx::get_journal_policy() const {
+    assert(snap_lock.is_locked());
+    assert(journal_policy != nullptr);
+    return journal_policy;
+  }
+
+  void ImageCtx::set_journal_policy(journal::Policy *policy) {
+    assert(snap_lock.is_wlocked());
+    assert(policy != nullptr);
+    delete journal_policy;
+    journal_policy = policy;
+  }
 }

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 98dcd42d4d3da28bd06ab549a7c4b4123034badc..551c4bfc22addbc1f2b8458a34d6fefc1e68b701 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -53,6 +53,7 @@ namespace librbd {
   template <typename> class Operations;
 
   namespace exclusive_lock { struct Policy; }
+  namespace journal { struct Policy; }
 
   namespace operation {
   template <typename> class ResizeRequest;
@@ -188,6 +189,7 @@ namespace librbd {
     LibrbdAdminSocketHook *asok_hook;
 
     exclusive_lock::Policy *exclusive_lock_policy = nullptr;
+    journal::Policy *journal_policy = nullptr;
 
     static bool _filter_metadata_confs(const string &prefix, std::map<string, bool> &configs,
                                        map<string, bufferlist> &pairs, map<string, bufferlist> *res);
@@ -288,6 +290,9 @@ namespace librbd {
 
     exclusive_lock::Policy *get_exclusive_lock_policy() const;
     void set_exclusive_lock_policy(exclusive_lock::Policy *policy);
+
+    journal::Policy *get_journal_policy() const;
+    void set_journal_policy(journal::Policy *policy);
   };
 }
 

diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am
index 233ec80d98bc468b076bcf6d112a9fda6283a9e8..5cdd9fbd159aed9f1d8389ee365792da59918a68 100644 (file)
--- a/src/librbd/Makefile.am
+++ b/src/librbd/Makefile.am
@@ -39,6 +39,7 @@ librbd_internal_la_SOURCES = \
        librbd/image_watcher/Notifier.cc \
        librbd/image_watcher/NotifyLockOwner.cc \
        librbd/journal/Replay.cc \
+       librbd/journal/StandardPolicy.cc \
        librbd/object_map/InvalidateRequest.cc \
        librbd/object_map/LockRequest.cc \
        librbd/object_map/Request.cc \
@@ -121,7 +122,9 @@ noinst_HEADERS += \
        librbd/image/SetSnapRequest.h \
        librbd/image_watcher/Notifier.h \
        librbd/image_watcher/NotifyLockOwner.h \
+       librbd/journal/Policy.h \
        librbd/journal/Replay.h \
+       librbd/journal/StandardPolicy.h \
        librbd/journal/Types.h \
        librbd/journal/TypeTraits.h \
        librbd/object_map/InvalidateRequest.h \

diff --git a/src/librbd/exclusive_lock/AcquireRequest.cc b/src/librbd/exclusive_lock/AcquireRequest.cc
index d973bf21fc624011a198cdefc4bb7efed1e37b2a..6ec148e40ef9c5be8a7f88192452a187b69a6d5b 100644 (file)
--- a/src/librbd/exclusive_lock/AcquireRequest.cc
+++ b/src/librbd/exclusive_lock/AcquireRequest.cc
@@ -14,6 +14,7 @@
 #include "librbd/Journal.h"
 #include "librbd/ObjectMap.h"
 #include "librbd/Utils.h"
+#include "librbd/journal/Policy.h"
 
 #define dout_subsys ceph_subsys_rbd
 #undef dout_prefix
@@ -151,7 +152,7 @@ Context *AcquireRequest<I>::send_open_journal() {
     this);
   m_journal = m_image_ctx.create_journal();
 
-  // journal playback required object map (if enabled) and itself
+  // journal playback requires object map (if enabled) and itself
   apply();
 
   m_journal->open(ctx);
@@ -179,17 +180,11 @@ void AcquireRequest<I>::send_allocate_journal_tag() {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 10) << __func__ << dendl;
 
-  if (!m_journal->is_tag_owner()) {
-    lderr(cct) << "local image not promoted" << dendl;
-    m_error_result = -EPERM;
-    send_close_journal();
-    return;
-  }
-
+  RWLock::RLocker snap_locker(m_image_ctx.snap_lock);
   using klass = AcquireRequest<I>;
   Context *ctx = create_context_callback<
     klass, &klass::handle_allocate_journal_tag>(this);
-  m_journal->allocate_tag(Journal<I>::LOCAL_MIRROR_UUID, ctx);
+  m_image_ctx.get_journal_policy()->allocate_tag_on_lock(ctx);
 }
 
 template <typename I>
@@ -198,6 +193,8 @@ Context *AcquireRequest<I>::handle_allocate_journal_tag(int *ret_val) {
   ldout(cct, 10) << __func__ << ": r=" << *ret_val << dendl;
 
   if (*ret_val < 0) {
+    lderr(cct) << "failed to allocate journal tag: " << cpp_strerror(*ret_val)
+               << dendl;
     m_error_result = *ret_val;
     send_close_journal();
     return nullptr;

diff --git a/src/librbd/journal/Policy.h b/src/librbd/journal/Policy.h
new file mode 100644 (file)
index 0000000..8265622
--- /dev/null
+++ b/src/librbd/journal/Policy.h
@@ -0,0 +1,24 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_JOURNAL_POLICY_H
+#define CEPH_LIBRBD_JOURNAL_POLICY_H
+
+class Context;
+
+namespace librbd {
+
+namespace journal {
+
+struct Policy {
+  virtual ~Policy() {
+  }
+
+  virtual void allocate_tag_on_lock(Context *on_finish) = 0;
+  virtual void cancel_external_replay(Context *on_finish) = 0;
+};
+
+} // namespace journal
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_JOURNAL_POLICY_H

diff --git a/src/librbd/journal/StandardPolicy.cc b/src/librbd/journal/StandardPolicy.cc
new file mode 100644 (file)
index 0000000..3f7e3df
--- /dev/null
+++ b/src/librbd/journal/StandardPolicy.cc
@@ -0,0 +1,34 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/journal/StandardPolicy.h"
+#include "common/WorkQueue.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Journal.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::journal::StandardPolicy: "
+
+namespace librbd {
+namespace journal {
+
+void StandardPolicy::allocate_tag_on_lock(Context *on_finish) {
+  assert(m_image_ctx->journal != nullptr);
+
+  if (!m_image_ctx->journal->is_tag_owner()) {
+    lderr(m_image_ctx->cct) << "local image not promoted" << dendl;
+    m_image_ctx->op_work_queue->queue(on_finish, -EPERM);
+    return;
+  }
+
+  m_image_ctx->journal->allocate_tag(Journal<>::LOCAL_MIRROR_UUID, on_finish);
+}
+
+void StandardPolicy::cancel_external_replay(Context *on_finish) {
+  // external replay is only handled by rbd-mirror
+  assert(false);
+}
+
+} // namespace journal
+} // namespace librbd

diff --git a/src/librbd/journal/StandardPolicy.h b/src/librbd/journal/StandardPolicy.h
new file mode 100644 (file)
index 0000000..c49ec9c
--- /dev/null
+++ b/src/librbd/journal/StandardPolicy.h
@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_JOURNAL_STANDARD_POLICY_H
+#define CEPH_LIBRBD_JOURNAL_STANDARD_POLICY_H
+
+#include "librbd/journal/Policy.h"
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace journal {
+
+class StandardPolicy : public Policy {
+public:
+  StandardPolicy(ImageCtx *image_ctx) : m_image_ctx(image_ctx) {
+  }
+
+  virtual void allocate_tag_on_lock(Context *on_finish);
+  virtual void cancel_external_replay(Context *on_finish);
+
+private:
+  ImageCtx *m_image_ctx;
+};
+
+} // namespace journal
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_JOURNAL_STANDARD_POLICY_H

diff --git a/src/test/Makefile-client.am b/src/test/Makefile-client.am
index b2ad5e8121b48b8e0e2313bf3e5fb282fbd5e12f..82d57a4802c0aca344fe4eeea8fa8b60bb6ff8f2 100644 (file)
--- a/src/test/Makefile-client.am
+++ b/src/test/Makefile-client.am
@@ -434,6 +434,7 @@ noinst_HEADERS += \
        test/librbd/mock/MockImageState.h \
        test/librbd/mock/MockImageWatcher.h \
        test/librbd/mock/MockJournal.h \
+       test/librbd/mock/MockJournalPolicy.h \
        test/librbd/mock/MockObjectMap.h \
        test/librbd/mock/MockOperations.h \
        test/librbd/mock/MockReadahead.h \

diff --git a/src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc b/src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc
index 9d67982f8c9691c5176d7ea271e4320b48bf7a95..c7c9428fed11f7d9f8dc723ae6d8d4183de9a647 100644 (file)
--- a/src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc
+++ b/src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc
@@ -5,6 +5,7 @@
 #include "test/librbd/test_support.h"
 #include "test/librbd/mock/MockImageCtx.h"
 #include "test/librbd/mock/MockJournal.h"
+#include "test/librbd/mock/MockJournalPolicy.h"
 #include "test/librbd/mock/MockObjectMap.h"
 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
 #include "test/librados_test_stub/MockTestMemRadosClient.h"
@@ -87,14 +88,17 @@ public:
                   .WillOnce(CompleteContext(0, mock_image_ctx.image_ctx->op_work_queue));
   }
 
-  void expect_is_journal_tag_owner(MockJournal &mock_journal, bool owner) {
-    EXPECT_CALL(mock_journal, is_tag_owner()).WillOnce(Return(owner));
+  void expect_get_journal_policy(MockImageCtx &mock_image_ctx,
+                                 MockJournalPolicy &mock_journal_policy) {
+    EXPECT_CALL(mock_image_ctx, get_journal_policy())
+                  .WillOnce(Return(&mock_journal_policy));
   }
 
   void expect_allocate_journal_tag(MockImageCtx &mock_image_ctx,
-                                   MockJournal &mock_journal, int r) {
-    EXPECT_CALL(mock_journal, allocate_tag("", _))
-                  .WillOnce(WithArg<1>(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)));
+                                   MockJournalPolicy &mock_journal_policy,
+                                   int r) {
+    EXPECT_CALL(mock_journal_policy, allocate_tag_on_lock(_))
+                  .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue));
   }
 
   void expect_get_lock_info(MockImageCtx &mock_image_ctx, int r,
@@ -185,11 +189,12 @@ TEST_F(TestMockExclusiveLockAcquireRequest, Success) {
   expect_open_object_map(mock_image_ctx, mock_object_map);
 
   MockJournal mock_journal;
+  MockJournalPolicy mock_journal_policy;
   expect_test_features(mock_image_ctx, RBD_FEATURE_JOURNALING, true);
   expect_create_journal(mock_image_ctx, &mock_journal);
   expect_open_journal(mock_image_ctx, mock_journal, 0);
-  expect_is_journal_tag_owner(mock_journal, true);
-  expect_allocate_journal_tag(mock_image_ctx, mock_journal, 0);
+  expect_get_journal_policy(mock_image_ctx, mock_journal_policy);
+  expect_allocate_journal_tag(mock_image_ctx, mock_journal_policy, 0);
 
   C_SaferCond acquire_ctx;
   C_SaferCond ctx;
@@ -247,11 +252,12 @@ TEST_F(TestMockExclusiveLockAcquireRequest, SuccessObjectMapDisabled) {
   expect_test_features(mock_image_ctx, RBD_FEATURE_OBJECT_MAP, false);
 
   MockJournal mock_journal;
+  MockJournalPolicy mock_journal_policy;
   expect_test_features(mock_image_ctx, RBD_FEATURE_JOURNALING, true);
   expect_create_journal(mock_image_ctx, &mock_journal);
   expect_open_journal(mock_image_ctx, mock_journal, 0);
-  expect_is_journal_tag_owner(mock_journal, true);
-  expect_allocate_journal_tag(mock_image_ctx, mock_journal, 0);
+  expect_get_journal_policy(mock_image_ctx, mock_journal_policy);
+  expect_allocate_journal_tag(mock_image_ctx, mock_journal_policy, 0);
 
   C_SaferCond acquire_ctx;
   C_SaferCond ctx;
@@ -297,41 +303,6 @@ TEST_F(TestMockExclusiveLockAcquireRequest, JournalError) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
-TEST_F(TestMockExclusiveLockAcquireRequest, NotJournalTagOwner) {
-  REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
-
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_flush_notifies(mock_image_ctx);
-  expect_lock(mock_image_ctx, 0);
-
-  MockObjectMap *mock_object_map = new MockObjectMap();
-  expect_test_features(mock_image_ctx, RBD_FEATURE_OBJECT_MAP, true);
-  expect_create_object_map(mock_image_ctx, mock_object_map);
-  expect_open_object_map(mock_image_ctx, *mock_object_map);
-
-  MockJournal *mock_journal = new MockJournal();
-  expect_test_features(mock_image_ctx, RBD_FEATURE_JOURNALING, true);
-  expect_create_journal(mock_image_ctx, mock_journal);
-  expect_open_journal(mock_image_ctx, *mock_journal, 0);
-  expect_is_journal_tag_owner(*mock_journal, false);
-  expect_close_journal(mock_image_ctx, *mock_journal);
-  expect_close_object_map(mock_image_ctx, *mock_object_map);
-
-  C_SaferCond acquire_ctx;
-  C_SaferCond ctx;
-  MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx,
-                                                       TEST_COOKIE,
-                                                       &acquire_ctx, &ctx);
-  req->send();
-  ASSERT_EQ(-EPERM, ctx.wait());
-}
-
 TEST_F(TestMockExclusiveLockAcquireRequest, AllocateJournalTagError) {
   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 
@@ -351,11 +322,12 @@ TEST_F(TestMockExclusiveLockAcquireRequest, AllocateJournalTagError) {
   expect_open_object_map(mock_image_ctx, *mock_object_map);
 
   MockJournal *mock_journal = new MockJournal();
+  MockJournalPolicy mock_journal_policy;
   expect_test_features(mock_image_ctx, RBD_FEATURE_JOURNALING, true);
   expect_create_journal(mock_image_ctx, mock_journal);
   expect_open_journal(mock_image_ctx, *mock_journal, 0);
-  expect_is_journal_tag_owner(*mock_journal, true);
-  expect_allocate_journal_tag(mock_image_ctx, *mock_journal, -ESTALE);
+  expect_get_journal_policy(mock_image_ctx, mock_journal_policy);
+  expect_allocate_journal_tag(mock_image_ctx, mock_journal_policy, -EPERM);
   expect_close_journal(mock_image_ctx, *mock_journal);
   expect_close_object_map(mock_image_ctx, *mock_object_map);
 
@@ -365,7 +337,7 @@ TEST_F(TestMockExclusiveLockAcquireRequest, AllocateJournalTagError) {
                                                        TEST_COOKIE,
                                                        &acquire_ctx, &ctx);
   req->send();
-  ASSERT_EQ(-ESTALE, ctx.wait());
+  ASSERT_EQ(-EPERM, ctx.wait());
 }
 
 TEST_F(TestMockExclusiveLockAcquireRequest, LockBusy) {

diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index 60d5fbbf996f79f53d23c0791bdcfe718f81b102..e48618c483ebd5cffa0f824647bad6d27e4e934f 100644 (file)
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -148,6 +148,8 @@ struct MockImageCtx {
   MOCK_METHOD0(notify_update, void());
   MOCK_METHOD1(notify_update, void(Context *));
 
+  MOCK_CONST_METHOD0(get_journal_policy, journal::Policy*());
+
   ImageCtx *image_ctx;
   CephContext *cct;
 

diff --git a/src/test/librbd/mock/MockJournalPolicy.h b/src/test/librbd/mock/MockJournalPolicy.h
new file mode 100644 (file)
index 0000000..e7debfa
--- /dev/null
+++ b/src/test/librbd/mock/MockJournalPolicy.h
@@ -0,0 +1,21 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_LIBRBD_MOCK_JOURNAL_POLICY_H
+#define CEPH_TEST_LIBRBD_MOCK_JOURNAL_POLICY_H
+
+#include "librbd/journal/Policy.h"
+#include "gmock/gmock.h"
+
+namespace librbd {
+
+struct MockJournalPolicy : public journal::Policy {
+
+  MOCK_METHOD1(allocate_tag_on_lock, void(Context*));
+  MOCK_METHOD1(cancel_external_replay, void(Context*));
+
+};
+
+} // namespace librbd
+
+#endif // CEPH_TEST_LIBRBD_MOCK_JOURNAL_POLICY_H