]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
librbd: helper state machine for asynchronous watch recovery
authorJason Dillaman <dillaman@redhat.com>
Mon, 15 Aug 2016 19:46:23 +0000 (15:46 -0400)
committerJason Dillaman <dillaman@redhat.com>
Mon, 30 Jan 2017 14:20:36 +0000 (09:20 -0500)
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 32180aaf42050a01981c33f84edd95eff931ee6c)

Conflicts:
src/librbd/CMakeLists.txt: trivial resolution
src/librbd/Makefile.am: trivial resolution

src/librbd/CMakeLists.txt
src/librbd/Makefile.am
src/librbd/image_watcher/RewatchRequest.cc [new file with mode: 0644]
src/librbd/image_watcher/RewatchRequest.h [new file with mode: 0644]
src/test/Makefile-client.am
src/test/librbd/CMakeLists.txt
src/test/librbd/image_watcher/test_mock_RewatchRequest.cc [new file with mode: 0644]
src/test/librbd/mock/MockExclusiveLock.h

index 8a09ae038bd94b59b38b64f54ce69026be0e5f03..3e9c4b5b9c184b4e34d19055a3176c2401dc73c3 100644 (file)
@@ -37,6 +37,7 @@ set(librbd_internal_srcs
   image/SetSnapRequest.cc
   image_watcher/Notifier.cc
   image_watcher/NotifyLockOwner.cc
+  image_watcher/RewatchRequest.cc
   journal/Replay.cc
   journal/StandardPolicy.cc
   object_map/InvalidateRequest.cc
index b9030e191a853f22c6aae5e5c0fee599dc851b28..002dfe7be01ea09f4f9a0192d156488a0d87e68b 100644 (file)
@@ -42,6 +42,7 @@ librbd_internal_la_SOURCES = \
        librbd/image/SetSnapRequest.cc \
        librbd/image_watcher/Notifier.cc \
        librbd/image_watcher/NotifyLockOwner.cc \
+       librbd/image_watcher/RewatchRequest.cc \
        librbd/journal/Replay.cc \
        librbd/journal/StandardPolicy.cc \
        librbd/object_map/InvalidateRequest.cc \
@@ -130,6 +131,7 @@ noinst_HEADERS += \
        librbd/image/SetSnapRequest.h \
        librbd/image_watcher/Notifier.h \
        librbd/image_watcher/NotifyLockOwner.h \
+       librbd/image_watcher/RewatchRequest.h \
        librbd/journal/DisabledPolicy.h \
        librbd/journal/Policy.h \
        librbd/journal/Replay.h \
diff --git a/src/librbd/image_watcher/RewatchRequest.cc b/src/librbd/image_watcher/RewatchRequest.cc
new file mode 100644 (file)
index 0000000..4d808f4
--- /dev/null
@@ -0,0 +1,125 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/image_watcher/RewatchRequest.h"
+#include "common/errno.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::image_watcher::RewatchRequest: " \
+                           << this << ": " << __func__
+
+namespace librbd {
+namespace image_watcher {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_safe_callback;
+
+template <typename I>
+RewatchRequest<I>::RewatchRequest(I &image_ctx, RWLock &watch_lock,
+                                  librados::WatchCtx2 *watch_ctx,
+                                  uint64_t *watch_handle, Context *on_finish)
+  : m_image_ctx(image_ctx), m_watch_lock(watch_lock), m_watch_ctx(watch_ctx),
+    m_watch_handle(watch_handle), m_on_finish(on_finish) {
+}
+
+template <typename I>
+void RewatchRequest<I>::send() {
+  unwatch();
+}
+
+template <typename I>
+void RewatchRequest<I>::unwatch() {
+  assert(m_watch_lock.is_wlocked());
+  assert(*m_watch_handle != 0);
+
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  librados::AioCompletion *aio_comp = create_rados_safe_callback<
+    RewatchRequest<I>, &RewatchRequest<I>::handle_unwatch>(this);
+  int r = m_image_ctx.md_ctx.aio_unwatch(*m_watch_handle, aio_comp);
+  assert(r == 0);
+  aio_comp->release();
+
+  *m_watch_handle = 0;
+}
+
+template <typename I>
+void RewatchRequest<I>::handle_unwatch(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r == -EBLACKLISTED) {
+    lderr(cct) << "client blacklisted" << dendl;
+    finish(r);
+  } else if (r < 0) {
+    lderr(cct) << "failed to unwatch: " << cpp_strerror(r) << dendl;
+  }
+  rewatch();
+}
+
+template <typename I>
+void RewatchRequest<I>::rewatch() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  librados::AioCompletion *aio_comp = create_rados_safe_callback<
+    RewatchRequest<I>, &RewatchRequest<I>::handle_rewatch>(this);
+  int r = m_image_ctx.md_ctx.aio_watch(m_image_ctx.header_oid, aio_comp,
+                                       &m_rewatch_handle, m_watch_ctx);
+  assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void RewatchRequest<I>::handle_rewatch(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r == -EBLACKLISTED) {
+    lderr(cct) << "client blacklisted" << dendl;
+    finish(r);
+    return;
+  } else if (r == -ENOENT) {
+    ldout(cct, 5) << "image header deleted" << dendl;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    lderr(cct) << "failed to watch image header: " << cpp_strerror(r)
+               << dendl;
+    rewatch();
+    return;
+  }
+
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    *m_watch_handle = m_rewatch_handle;
+  }
+
+  {
+    RWLock::RLocker owner_locker(m_image_ctx.owner_lock);
+    if (m_image_ctx.exclusive_lock != nullptr) {
+      // update the lock cookie with the new watch handle
+      m_image_ctx.exclusive_lock->reacquire_lock();
+    }
+  }
+  finish(0);
+}
+
+template <typename I>
+void RewatchRequest<I>::finish(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_watcher
+} // namespace librbd
+
+template class librbd::image_watcher::RewatchRequest<librbd::ImageCtx>;
diff --git a/src/librbd/image_watcher/RewatchRequest.h b/src/librbd/image_watcher/RewatchRequest.h
new file mode 100644 (file)
index 0000000..dc3a2cd
--- /dev/null
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H
+#define CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H
+
+#include "include/int_types.h"
+#include "include/rados/librados.hpp"
+
+struct Context;
+struct RWLock;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace image_watcher {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class RewatchRequest {
+public:
+
+  static RewatchRequest *create(ImageCtxT &image_ctx, RWLock &watch_lock,
+                                librados::WatchCtx2 *watch_ctx,
+                                uint64_t *watch_handle, Context *on_finish) {
+    return new RewatchRequest(image_ctx, watch_lock, watch_ctx, watch_handle,
+                              on_finish);
+  }
+
+  RewatchRequest(ImageCtxT &image_ctx, RWLock &watch_lock,
+                 librados::WatchCtx2 *watch_ctx, uint64_t *watch_handle,
+                 Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * UNWATCH
+   *    |
+   *    |  . . . .
+   *    |  .     . (recoverable error)
+   *    v  v     .
+   * REWATCH . . .
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ImageCtxT &m_image_ctx;
+  RWLock &m_watch_lock;
+  librados::WatchCtx2 *m_watch_ctx;
+  uint64_t *m_watch_handle;
+  Context *m_on_finish;
+
+  uint64_t m_rewatch_handle = 0;
+
+  void unwatch();
+  void handle_unwatch(int r);
+
+  void rewatch();
+  void handle_rewatch(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_watcher
+} // namespace librbd
+
+extern template class librbd::image_watcher::RewatchRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H
index 28a365c9fbd0f2dde70a701700a48a561ec4bb78..18ff9ad2ae3fb00fcaa2d86e242a63c52905ddec 100644 (file)
@@ -393,6 +393,7 @@ unittest_librbd_SOURCES = \
        test/librbd/exclusive_lock/test_mock_ReacquireRequest.cc \
        test/librbd/exclusive_lock/test_mock_ReleaseRequest.cc \
        test/librbd/image/test_mock_RefreshRequest.cc \
+       test/librbd/image_watcher/test_mock_RewatchRequest.cc \
        test/librbd/journal/test_mock_Replay.cc \
        test/librbd/object_map/test_mock_InvalidateRequest.cc \
        test/librbd/object_map/test_mock_LockRequest.cc \
index 4e0f92b9581f19bd809129733d4e0d1c4ac83e59..a8ff68edbf65b13f4ec0b4e3afe61fe83b18f125 100644 (file)
@@ -27,6 +27,7 @@ set(unittest_librbd_srcs
   exclusive_lock/test_mock_ReacquireRequest.cc
   exclusive_lock/test_mock_ReleaseRequest.cc
   image/test_mock_RefreshRequest.cc
+  image_watcher/test_mock_RewatchRequest.cc
   journal/test_mock_Replay.cc
   object_map/test_mock_InvalidateRequest.cc
   object_map/test_mock_LockRequest.cc
diff --git a/src/test/librbd/image_watcher/test_mock_RewatchRequest.cc b/src/test/librbd/image_watcher/test_mock_RewatchRequest.cc
new file mode 100644 (file)
index 0000000..5609134
--- /dev/null
@@ -0,0 +1,215 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_mock_fixture.h"
+#include "include/rados/librados.hpp"
+#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
+#include "test/librados_test_stub/MockTestMemRadosClient.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockExclusiveLock.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "librados/AioCompletionImpl.h"
+#include "librbd/image_watcher/RewatchRequest.h"
+
+namespace librbd {
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+  MockTestImageCtx(ImageCtx &image_ctx) : MockImageCtx(image_ctx) {
+  }
+};
+
+} // anonymous namespace
+} // namespace librbd
+
+#include "librbd/image_watcher/RewatchRequest.cc"
+
+namespace librbd {
+namespace image_watcher {
+
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::WithArg;
+
+struct TestMockImageWatcherRewatchRequest : public TestMockFixture {
+  typedef RewatchRequest<librbd::MockTestImageCtx> MockRewatchRequest;
+
+  TestMockImageWatcherRewatchRequest()
+    : m_watch_lock("watch_lock") {
+  }
+
+  void expect_aio_watch(MockImageCtx &mock_image_ctx, int r) {
+    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(
+      mock_image_ctx.md_ctx));
+
+    EXPECT_CALL(mock_io_ctx, aio_watch(mock_image_ctx.header_oid, _, _, _))
+      .WillOnce(DoAll(WithArg<1>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c) {
+                                   c->get();
+                                   mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) {
+                                       mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r);
+                                     }), r);
+                                   })),
+                      Return(0)));
+  }
+
+  void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r) {
+    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(
+      mock_image_ctx.md_ctx));
+
+    EXPECT_CALL(mock_io_ctx, aio_unwatch(m_watch_handle, _))
+      .WillOnce(DoAll(Invoke([&mock_image_ctx, &mock_io_ctx, r](uint64_t handle,
+                                                                librados::AioCompletionImpl *c) {
+                        c->get();
+                        mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) {
+                            mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r);
+                          }), r);
+                        }),
+                      Return(0)));
+  }
+
+  void expect_reacquire_lock(MockExclusiveLock &mock_exclusive_lock) {
+    EXPECT_CALL(mock_exclusive_lock, reacquire_lock());
+  }
+
+  struct WatchCtx : public librados::WatchCtx2 {
+    virtual void handle_notify(uint64_t, uint64_t, uint64_t,
+                               ceph::bufferlist&) {
+      assert(false);
+    }
+    virtual void handle_error(uint64_t, int) {
+      assert(false);
+    }
+  };
+
+  RWLock m_watch_lock;
+  WatchCtx m_watch_ctx;
+  uint64_t m_watch_handle = 123;
+};
+
+TEST_F(TestMockImageWatcherRewatchRequest, Success) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, 0);
+
+  MockExclusiveLock mock_exclusive_lock;
+  if (ictx->test_features(RBD_FEATURE_EXCLUSIVE_LOCK)) {
+    mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+    expect_reacquire_lock(mock_exclusive_lock);
+  }
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+  ASSERT_EQ(0, ctx.wait());
+}
+
+TEST_F(TestMockImageWatcherRewatchRequest, UnwatchError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_unwatch(mock_image_ctx, -EINVAL);
+  expect_aio_watch(mock_image_ctx, 0);
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+  ASSERT_EQ(0, ctx.wait());
+}
+
+TEST_F(TestMockImageWatcherRewatchRequest, WatchBlacklist) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, -EBLACKLISTED);
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+  ASSERT_EQ(-EBLACKLISTED, ctx.wait());
+}
+
+TEST_F(TestMockImageWatcherRewatchRequest, WatchDNE) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, -ENOENT);
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+  ASSERT_EQ(-ENOENT, ctx.wait());
+}
+
+TEST_F(TestMockImageWatcherRewatchRequest, WatchError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, -EINVAL);
+  expect_aio_watch(mock_image_ctx, 0);
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+  ASSERT_EQ(0, ctx.wait());
+}
+
+} // namespace image_watcher
+} // namespace librbd
index 97f9da12fbc52e10ae0f1ca879a1b90ec64d244f..83d3058d1d8bd7a34504d9dd7079a764f7ef6382 100644 (file)
@@ -19,6 +19,8 @@ struct MockExclusiveLock {
 
   MOCK_METHOD2(init, void(uint64_t features, Context*));
   MOCK_METHOD1(shut_down, void(Context*));
+
+  MOCK_METHOD0(reacquire_lock, void());
 };
 
 } // namespace librbd