From 62f265b30531141dfda8a7490d18b1d0d787fe13 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 15 Aug 2016 15:46:23 -0400 Subject: [PATCH] librbd: helper state machine for asynchronous watch recovery Signed-off-by: Jason Dillaman (cherry picked from commit 32180aaf42050a01981c33f84edd95eff931ee6c) Conflicts: src/librbd/CMakeLists.txt: trivial resolution src/librbd/Makefile.am: trivial resolution --- src/librbd/CMakeLists.txt | 1 + src/librbd/Makefile.am | 2 + src/librbd/image_watcher/RewatchRequest.cc | 125 ++++++++++ src/librbd/image_watcher/RewatchRequest.h | 78 +++++++ src/test/Makefile-client.am | 1 + src/test/librbd/CMakeLists.txt | 1 + .../image_watcher/test_mock_RewatchRequest.cc | 215 ++++++++++++++++++ src/test/librbd/mock/MockExclusiveLock.h | 2 + 8 files changed, 425 insertions(+) create mode 100644 src/librbd/image_watcher/RewatchRequest.cc create mode 100644 src/librbd/image_watcher/RewatchRequest.h create mode 100644 src/test/librbd/image_watcher/test_mock_RewatchRequest.cc diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 8a09ae038bd94..3e9c4b5b9c184 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -37,6 +37,7 @@ set(librbd_internal_srcs image/SetSnapRequest.cc image_watcher/Notifier.cc image_watcher/NotifyLockOwner.cc + image_watcher/RewatchRequest.cc journal/Replay.cc journal/StandardPolicy.cc object_map/InvalidateRequest.cc diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am index b9030e191a853..002dfe7be01ea 100644 --- a/src/librbd/Makefile.am +++ b/src/librbd/Makefile.am @@ -42,6 +42,7 @@ librbd_internal_la_SOURCES = \ librbd/image/SetSnapRequest.cc \ librbd/image_watcher/Notifier.cc \ librbd/image_watcher/NotifyLockOwner.cc \ + librbd/image_watcher/RewatchRequest.cc \ librbd/journal/Replay.cc \ librbd/journal/StandardPolicy.cc \ librbd/object_map/InvalidateRequest.cc \ @@ -130,6 +131,7 @@ noinst_HEADERS += \ librbd/image/SetSnapRequest.h \ librbd/image_watcher/Notifier.h \ librbd/image_watcher/NotifyLockOwner.h \ + librbd/image_watcher/RewatchRequest.h \ librbd/journal/DisabledPolicy.h \ librbd/journal/Policy.h \ librbd/journal/Replay.h \ diff --git a/src/librbd/image_watcher/RewatchRequest.cc b/src/librbd/image_watcher/RewatchRequest.cc new file mode 100644 index 0000000000000..4d808f459c5d3 --- /dev/null +++ b/src/librbd/image_watcher/RewatchRequest.cc @@ -0,0 +1,125 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image_watcher/RewatchRequest.h" +#include "common/errno.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image_watcher::RewatchRequest: " \ + << this << ": " << __func__ + +namespace librbd { +namespace image_watcher { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_safe_callback; + +template +RewatchRequest::RewatchRequest(I &image_ctx, RWLock &watch_lock, + librados::WatchCtx2 *watch_ctx, + uint64_t *watch_handle, Context *on_finish) + : m_image_ctx(image_ctx), m_watch_lock(watch_lock), m_watch_ctx(watch_ctx), + m_watch_handle(watch_handle), m_on_finish(on_finish) { +} + +template +void RewatchRequest::send() { + unwatch(); +} + +template +void RewatchRequest::unwatch() { + assert(m_watch_lock.is_wlocked()); + assert(*m_watch_handle != 0); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + librados::AioCompletion *aio_comp = create_rados_safe_callback< + RewatchRequest, &RewatchRequest::handle_unwatch>(this); + int r = m_image_ctx.md_ctx.aio_unwatch(*m_watch_handle, aio_comp); + assert(r == 0); + aio_comp->release(); + + *m_watch_handle = 0; +} + +template +void RewatchRequest::handle_unwatch(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r == -EBLACKLISTED) { + lderr(cct) << "client blacklisted" << dendl; + finish(r); + } else if (r < 0) { + lderr(cct) << "failed to unwatch: " << cpp_strerror(r) << dendl; + } + rewatch(); +} + +template +void RewatchRequest::rewatch() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + librados::AioCompletion *aio_comp = create_rados_safe_callback< + RewatchRequest, &RewatchRequest::handle_rewatch>(this); + int r = m_image_ctx.md_ctx.aio_watch(m_image_ctx.header_oid, aio_comp, + &m_rewatch_handle, m_watch_ctx); + assert(r == 0); + aio_comp->release(); +} + +template +void RewatchRequest::handle_rewatch(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r == -EBLACKLISTED) { + lderr(cct) << "client blacklisted" << dendl; + finish(r); + return; + } else if (r == -ENOENT) { + ldout(cct, 5) << "image header deleted" << dendl; + finish(r); + return; + } else if (r < 0) { + lderr(cct) << "failed to watch image header: " << cpp_strerror(r) + << dendl; + rewatch(); + return; + } + + { + RWLock::WLocker watch_locker(m_watch_lock); + *m_watch_handle = m_rewatch_handle; + } + + { + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + if (m_image_ctx.exclusive_lock != nullptr) { + // update the lock cookie with the new watch handle + m_image_ctx.exclusive_lock->reacquire_lock(); + } + } + finish(0); +} + +template +void RewatchRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_watcher +} // namespace librbd + +template class librbd::image_watcher::RewatchRequest; diff --git a/src/librbd/image_watcher/RewatchRequest.h b/src/librbd/image_watcher/RewatchRequest.h new file mode 100644 index 0000000000000..dc3a2cdfd5379 --- /dev/null +++ b/src/librbd/image_watcher/RewatchRequest.h @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H +#define CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" + +struct Context; +struct RWLock; + +namespace librbd { + +class ImageCtx; + +namespace image_watcher { + +template +class RewatchRequest { +public: + + static RewatchRequest *create(ImageCtxT &image_ctx, RWLock &watch_lock, + librados::WatchCtx2 *watch_ctx, + uint64_t *watch_handle, Context *on_finish) { + return new RewatchRequest(image_ctx, watch_lock, watch_ctx, watch_handle, + on_finish); + } + + RewatchRequest(ImageCtxT &image_ctx, RWLock &watch_lock, + librados::WatchCtx2 *watch_ctx, uint64_t *watch_handle, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * UNWATCH + * | + * | . . . . + * | . . (recoverable error) + * v v . + * REWATCH . . . + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + RWLock &m_watch_lock; + librados::WatchCtx2 *m_watch_ctx; + uint64_t *m_watch_handle; + Context *m_on_finish; + + uint64_t m_rewatch_handle = 0; + + void unwatch(); + void handle_unwatch(int r); + + void rewatch(); + void handle_rewatch(int r); + + void finish(int r); +}; + +} // namespace image_watcher +} // namespace librbd + +extern template class librbd::image_watcher::RewatchRequest; + +#endif // CEPH_LIBRBD_IMAGE_WATCHER_REWATCH_REQUEST_H diff --git a/src/test/Makefile-client.am b/src/test/Makefile-client.am index 28a365c9fbd0f..18ff9ad2ae3fb 100644 --- a/src/test/Makefile-client.am +++ b/src/test/Makefile-client.am @@ -393,6 +393,7 @@ unittest_librbd_SOURCES = \ test/librbd/exclusive_lock/test_mock_ReacquireRequest.cc \ test/librbd/exclusive_lock/test_mock_ReleaseRequest.cc \ test/librbd/image/test_mock_RefreshRequest.cc \ + test/librbd/image_watcher/test_mock_RewatchRequest.cc \ test/librbd/journal/test_mock_Replay.cc \ test/librbd/object_map/test_mock_InvalidateRequest.cc \ test/librbd/object_map/test_mock_LockRequest.cc \ diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 4e0f92b9581f1..a8ff68edbf65b 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -27,6 +27,7 @@ set(unittest_librbd_srcs exclusive_lock/test_mock_ReacquireRequest.cc exclusive_lock/test_mock_ReleaseRequest.cc image/test_mock_RefreshRequest.cc + image_watcher/test_mock_RewatchRequest.cc journal/test_mock_Replay.cc object_map/test_mock_InvalidateRequest.cc object_map/test_mock_LockRequest.cc diff --git a/src/test/librbd/image_watcher/test_mock_RewatchRequest.cc b/src/test/librbd/image_watcher/test_mock_RewatchRequest.cc new file mode 100644 index 0000000000000..5609134c54456 --- /dev/null +++ b/src/test/librbd/image_watcher/test_mock_RewatchRequest.cc @@ -0,0 +1,215 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "include/rados/librados.hpp" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/test_support.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "librados/AioCompletionImpl.h" +#include "librbd/image_watcher/RewatchRequest.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(ImageCtx &image_ctx) : MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +#include "librbd/image_watcher/RewatchRequest.cc" + +namespace librbd { +namespace image_watcher { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::WithArg; + +struct TestMockImageWatcherRewatchRequest : public TestMockFixture { + typedef RewatchRequest MockRewatchRequest; + + TestMockImageWatcherRewatchRequest() + : m_watch_lock("watch_lock") { + } + + void expect_aio_watch(MockImageCtx &mock_image_ctx, int r) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx( + mock_image_ctx.md_ctx)); + + EXPECT_CALL(mock_io_ctx, aio_watch(mock_image_ctx.header_oid, _, _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c) { + c->get(); + mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) { + mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r); + }), r); + })), + Return(0))); + } + + void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx( + mock_image_ctx.md_ctx)); + + EXPECT_CALL(mock_io_ctx, aio_unwatch(m_watch_handle, _)) + .WillOnce(DoAll(Invoke([&mock_image_ctx, &mock_io_ctx, r](uint64_t handle, + librados::AioCompletionImpl *c) { + c->get(); + mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) { + mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r); + }), r); + }), + Return(0))); + } + + void expect_reacquire_lock(MockExclusiveLock &mock_exclusive_lock) { + EXPECT_CALL(mock_exclusive_lock, reacquire_lock()); + } + + struct WatchCtx : public librados::WatchCtx2 { + virtual void handle_notify(uint64_t, uint64_t, uint64_t, + ceph::bufferlist&) { + assert(false); + } + virtual void handle_error(uint64_t, int) { + assert(false); + } + }; + + RWLock m_watch_lock; + WatchCtx m_watch_ctx; + uint64_t m_watch_handle = 123; +}; + +TEST_F(TestMockImageWatcherRewatchRequest, Success) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, 0); + + MockExclusiveLock mock_exclusive_lock; + if (ictx->test_features(RBD_FEATURE_EXCLUSIVE_LOCK)) { + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_reacquire_lock(mock_exclusive_lock); + } + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageWatcherRewatchRequest, UnwatchError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_unwatch(mock_image_ctx, -EINVAL); + expect_aio_watch(mock_image_ctx, 0); + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageWatcherRewatchRequest, WatchBlacklist) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, -EBLACKLISTED); + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); +} + +TEST_F(TestMockImageWatcherRewatchRequest, WatchDNE) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, -ENOENT); + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageWatcherRewatchRequest, WatchError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, -EINVAL); + expect_aio_watch(mock_image_ctx, 0); + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_watcher +} // namespace librbd diff --git a/src/test/librbd/mock/MockExclusiveLock.h b/src/test/librbd/mock/MockExclusiveLock.h index 97f9da12fbc52..83d3058d1d8bd 100644 --- a/src/test/librbd/mock/MockExclusiveLock.h +++ b/src/test/librbd/mock/MockExclusiveLock.h @@ -19,6 +19,8 @@ struct MockExclusiveLock { MOCK_METHOD2(init, void(uint64_t features, Context*)); MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD0(reacquire_lock, void()); }; } // namespace librbd -- 2.39.5