From: Jason Dillaman Date: Thu, 30 Aug 2018 20:51:10 +0000 (-0400) Subject: librbd: attempt to recover lost image watcher upon all failures X-Git-Tag: v13.2.3~133^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=38c8b912b9ea314e1a53af6c9f6a464f40ac3bd1;p=ceph.git librbd: attempt to recover lost image watcher upon all failures For example, if an image is blacklisted and the blacklist eventually expires, the image should recover its watch. Fixes: http://tracker.ceph.com/issues/34534 Signed-off-by: Jason Dillaman (cherry picked from commit 23b7447f6be87a14f84664f29431d2fdd2af4512) Conflicts: src/librbd/watcher/RewatchRequest.cc: trivial resolution src/test/librbd/test_mock_Watcher.cc: trivial resolution --- diff --git a/src/librbd/Watcher.cc b/src/librbd/Watcher.cc index 9a72fc13a76b..1865eaaf8420 100644 --- a/src/librbd/Watcher.cc +++ b/src/librbd/Watcher.cc @@ -314,9 +314,9 @@ void Watcher::handle_rewatch_callback(int r) { if (m_unregister_watch_ctx != nullptr) { m_watch_state = WATCH_STATE_IDLE; std::swap(unregister_watch_ctx, m_unregister_watch_ctx); - } else if (r == -EBLACKLISTED || r == -ENOENT) { + } else if (r == -ENOENT) { m_watch_state = WATCH_STATE_IDLE; - } else if (m_watch_error) { + } else if (r < 0 || m_watch_error) { watch_error = true; } else { m_watch_state = WATCH_STATE_IDLE; diff --git a/src/librbd/watcher/RewatchRequest.cc b/src/librbd/watcher/RewatchRequest.cc index 22d2bb8e5678..48b965110ccd 100644 --- a/src/librbd/watcher/RewatchRequest.cc +++ b/src/librbd/watcher/RewatchRequest.cc @@ -35,7 +35,11 @@ void RewatchRequest::send() { void RewatchRequest::unwatch() { assert(m_watch_lock.is_wlocked()); - assert(*m_watch_handle != 0); + ceph_assert(m_watch_lock.is_wlocked()); + if (*m_watch_handle == 0) { + rewatch(); + return; + } CephContext *cct = reinterpret_cast(m_ioctx.cct()); ldout(cct, 10) << dendl; @@ -81,8 +85,7 @@ void RewatchRequest::handle_rewatch(int r) { if (r < 0) { lderr(cct) << "failed to watch object: " << cpp_strerror(r) << dendl; - finish(r); - return; + m_rewatch_handle = 0; } { @@ -90,7 +93,7 @@ void RewatchRequest::handle_rewatch(int r) { *m_watch_handle = m_rewatch_handle; } - finish(0); + finish(r); } void RewatchRequest::finish(int r) { diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 667cbdfab00b..924daeecaf63 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -34,6 +34,7 @@ set(unittest_librbd_srcs test_mock_ManagedLock.cc test_mock_ObjectMap.cc test_mock_TrashWatcher.cc + test_mock_Watcher.cc deep_copy/test_mock_ImageCopyRequest.cc deep_copy/test_mock_MetadataCopyRequest.cc deep_copy/test_mock_ObjectCopyRequest.cc diff --git a/src/test/librbd/test_mock_ObjectWatcher.cc b/src/test/librbd/test_mock_ObjectWatcher.cc deleted file mode 100644 index b17a55f3979e..000000000000 --- a/src/test/librbd/test_mock_ObjectWatcher.cc +++ /dev/null @@ -1,404 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab - -#include "test/librbd/test_mock_fixture.h" -#include "test/librbd/test_support.h" -#include "test/librbd/mock/MockImageCtx.h" -#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" -#include "test/librados_test_stub/MockTestMemRadosClient.h" -#include "common/Cond.h" -#include "common/Mutex.h" -#include "librados/AioCompletionImpl.h" -#include "librbd/ObjectWatcher.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include - -namespace librbd { - -namespace { - -struct MockObjectWatcher : public ObjectWatcher { - std::string oid; - - MockObjectWatcher(MockImageCtx &mock_image_ctx, const std::string &oid) - : ObjectWatcher(mock_image_ctx.md_ctx, - mock_image_ctx.op_work_queue), - oid(oid) { - } - - virtual std::string get_oid() const override { - return oid; - } - - virtual void handle_notify(uint64_t notify_id, uint64_t handle, - bufferlist &bl) { - } -}; - -} // anonymous namespace - -} // namespace librbd - -// template definitions -#include "librbd/ObjectWatcher.cc" -template class librbd::ObjectWatcher; - -namespace librbd { - -using ::testing::_; -using ::testing::DoDefault; -using ::testing::Invoke; -using ::testing::InSequence; -using ::testing::Return; -using ::testing::SaveArg; -using ::testing::WithArg; - -class TestMockObjectWatcher : public TestMockFixture { -public: - TestMockObjectWatcher() : m_lock("TestMockObjectWatcher::m_lock") { - } - - virtual void SetUp() { - TestMockFixture::SetUp(); - - m_oid = get_temp_image_name(); - - bufferlist bl; - ASSERT_EQ(0, m_ioctx.write_full(m_oid, bl)); - } - - void expect_aio_watch(MockImageCtx &mock_image_ctx, int r, - const std::function &action = std::function()) { - librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx( - mock_image_ctx.md_ctx)); - librados::MockTestMemRadosClient *mock_rados_client( - mock_io_ctx.get_mock_rados_client()); - - auto &expect = EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)); - if (r < 0) { - expect.WillOnce(DoAll(WithArg<1>(Invoke([this, mock_rados_client, r, action](librados::AioCompletionImpl *c) { - if (action) { - action(); - } - - c->get(); - mock_rados_client->finish_aio_completion(c, r); - notify_watch(); - })), - Return(0))); - } else { - expect.WillOnce(DoAll(SaveArg<3>(&m_watch_ctx), - Invoke([this, &mock_io_ctx, action](const std::string& o, - librados::AioCompletionImpl *c, - uint64_t *handle, - librados::WatchCtx2 *ctx) { - if (action) { - action(); - } - - mock_io_ctx.do_aio_watch(o, c, handle, ctx); - notify_watch(); - }), - Return(0))); - } - } - - void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r, - const std::function &action = std::function()) { - librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx( - mock_image_ctx.md_ctx)); - - auto &expect = EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)); - if (r < 0) { - expect.WillOnce(DoAll(Invoke([this, &mock_io_ctx, r, action](uint64_t handle, - librados::AioCompletionImpl *c) { - if (action) { - action(); - } - - librados::AioCompletionImpl *dummy_c = new librados::AioCompletionImpl(); - mock_io_ctx.do_aio_unwatch(handle, dummy_c); - ASSERT_EQ(0, dummy_c->wait_for_complete()); - dummy_c->release(); - - c->get(); - mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r); - notify_watch(); - }), - Return(0))); - } else { - expect.WillOnce(DoAll(Invoke([this, &mock_io_ctx, action](uint64_t handle, - librados::AioCompletionImpl *c) { - if (action) { - action(); - } - - mock_io_ctx.do_aio_unwatch(handle, c); - notify_watch(); - }), - Return(0))); - } - } - - std::string m_oid; - librados::WatchCtx2 *m_watch_ctx = nullptr; - - void notify_watch() { - Mutex::Locker locker(m_lock); - ++m_watch_count; - m_cond.Signal(); - } - - bool wait_for_watch(MockImageCtx &mock_image_ctx, size_t count) { - Mutex::Locker locker(m_lock); - while (m_watch_count < count) { - if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) { - return false; - } - } - return true; - } - - Mutex m_lock; - Cond m_cond; - size_t m_watch_count = 0; -}; - -TEST_F(TestMockObjectWatcher, Success) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - C_SaferCond unregister_ctx; - mock_image_watcher.unregister_watch(&unregister_ctx); - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, RegisterError) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - InSequence seq; - expect_aio_watch(mock_image_ctx, -EINVAL); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(-EINVAL, register_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, UnregisterError) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, -EINVAL); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - C_SaferCond unregister_ctx; - mock_image_watcher.unregister_watch(&unregister_ctx); - ASSERT_EQ(-EINVAL, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, Reregister) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - // wait for recovery unwatch/watch - ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3)); - - C_SaferCond unregister_ctx; - mock_image_watcher.unregister_watch(&unregister_ctx); - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, ReregisterUnwatchError) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, -EINVAL); - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - // wait for recovery unwatch/watch - ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3)); - - C_SaferCond unregister_ctx; - mock_image_watcher.unregister_watch(&unregister_ctx); - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, ReregisterWatchError) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - expect_aio_watch(mock_image_ctx, -ESHUTDOWN); - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - // wait for recovery unwatch/watch - ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4)); - - C_SaferCond unregister_ctx; - mock_image_watcher.unregister_watch(&unregister_ctx); - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, ReregisterUnwatchPendingUnregister) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - - // inject an unregister - C_SaferCond unregister_ctx; - expect_aio_unwatch(mock_image_ctx, 0, [&mock_image_watcher, &unregister_ctx]() { - mock_image_watcher.unregister_watch(&unregister_ctx); - }); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, ReregisterWatchPendingUnregister) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - // inject an unregister - C_SaferCond unregister_ctx; - expect_aio_watch(mock_image_ctx, -ESHUTDOWN, - [&mock_image_watcher, &unregister_ctx]() { - mock_image_watcher.unregister_watch(&unregister_ctx); - }); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - ASSERT_EQ(0, unregister_ctx.wait()); -} - -TEST_F(TestMockObjectWatcher, ReregisterPendingUnregister) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockImageCtx mock_image_ctx(*ictx); - MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid); - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_aio_watch(mock_image_ctx, 0); - expect_aio_unwatch(mock_image_ctx, 0); - - // inject an unregister - C_SaferCond unregister_ctx; - expect_aio_watch(mock_image_ctx, 0, - [&mock_image_watcher, &unregister_ctx]() { - mock_image_watcher.unregister_watch(&unregister_ctx); - }); - - expect_aio_unwatch(mock_image_ctx, 0); - - C_SaferCond register_ctx; - mock_image_watcher.register_watch(®ister_ctx); - ASSERT_EQ(0, register_ctx.wait()); - - assert(m_watch_ctx != nullptr); - m_watch_ctx->handle_error(0, -ESHUTDOWN); - - ASSERT_EQ(0, unregister_ctx.wait()); -} - -} // namespace librbd diff --git a/src/test/librbd/test_mock_Watcher.cc b/src/test/librbd/test_mock_Watcher.cc new file mode 100644 index 000000000000..4feffc67d68e --- /dev/null +++ b/src/test/librbd/test_mock_Watcher.cc @@ -0,0 +1,403 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "test/librbd/test_support.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "common/Cond.h" +#include "common/Mutex.h" +#include "librados/AioCompletionImpl.h" +#include "librbd/Watcher.h" +#include "librbd/watcher/RewatchRequest.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include + +namespace librbd { + +namespace { + +struct MockWatcher : public Watcher { + std::string oid; + + MockWatcher(librados::IoCtx& ioctx, ContextWQ *work_queue, + const std::string& oid) + : Watcher(ioctx, work_queue, oid) { + } + + virtual void handle_notify(uint64_t notify_id, uint64_t handle, + uint64_t notifier_id, bufferlist &bl) { + } +}; + +} // anonymous namespace +} // namespace librbd + +namespace librbd { + +using ::testing::_; +using ::testing::DoDefault; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::SaveArg; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockWatcher : public TestMockFixture { +public: + TestMockWatcher() : m_lock("TestMockWatcher::m_lock") { + } + + virtual void SetUp() { + TestMockFixture::SetUp(); + + m_oid = get_temp_image_name(); + + bufferlist bl; + ASSERT_EQ(0, m_ioctx.write_full(m_oid, bl)); + } + + void expect_aio_watch(MockImageCtx &mock_image_ctx, int r, + const std::function &action = std::function()) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + librados::MockTestMemRadosClient *mock_rados_client( + mock_io_ctx.get_mock_rados_client()); + + EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)) + .WillOnce(DoAll(WithArgs<1, 2, 3>(Invoke([this, &mock_image_ctx, mock_rados_client, r, action]( + librados::AioCompletionImpl *c, uint64_t *cookie, + librados::WatchCtx2 *watch_ctx) { + if (r == 0) { + *cookie = 234U; + m_watch_ctx = watch_ctx; + } + + c->get(); + mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([mock_rados_client, action, c](int r) { + if (action) { + action(); + } + + mock_rados_client->finish_aio_completion(c, r); + }), r); + notify_watch(); + })), Return(0))); + } + + void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r, + const std::function &action = std::function()) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + librados::MockTestMemRadosClient *mock_rados_client( + mock_io_ctx.get_mock_rados_client()); + + EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)) + .WillOnce(DoAll(Invoke([this, &mock_image_ctx, mock_rados_client, r, action]( + uint64_t handle, librados::AioCompletionImpl *c) { + c->get(); + mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([mock_rados_client, action, c](int r) { + if (action) { + action(); + } + + mock_rados_client->finish_aio_completion(c, r); + }), r); + notify_watch(); + }), Return(0))); + } + + std::string m_oid; + librados::WatchCtx2 *m_watch_ctx = nullptr; + + void notify_watch() { + Mutex::Locker locker(m_lock); + ++m_watch_count; + m_cond.Signal(); + } + + bool wait_for_watch(MockImageCtx &mock_image_ctx, size_t count) { + Mutex::Locker locker(m_lock); + while (m_watch_count < count) { + if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) { + return false; + } + } + return true; + } + + Mutex m_lock; + Cond m_cond; + size_t m_watch_count = 0; +}; + +TEST_F(TestMockWatcher, Success) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, RegisterError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + InSequence seq; + expect_aio_watch(mock_image_ctx, -EINVAL); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(-EINVAL, register_ctx.wait()); +} + +TEST_F(TestMockWatcher, UnregisterError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, -EINVAL); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(-EINVAL, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, Reregister) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -ESHUTDOWN); + + // wait for recovery unwatch/watch + ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3)); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterUnwatchError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, -EINVAL); + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -ESHUTDOWN); + + // wait for recovery unwatch/watch + ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3)); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterWatchError) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, -EPERM); + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -ESHUTDOWN); + + // wait for recovery unwatch/watch + ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4)); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterWatchBlacklist) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + expect_aio_watch(mock_image_ctx, -EBLACKLISTED); + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + ceph_assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -EBLACKLISTED); + + // wait for recovery unwatch/watch + ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4)); + + C_SaferCond unregister_ctx; + mock_image_watcher.unregister_watch(&unregister_ctx); + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterUnwatchPendingUnregister) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + + // inject an unregister + C_SaferCond unregister_ctx; + expect_aio_unwatch(mock_image_ctx, -EBLACKLISTED, + [&mock_image_watcher, &unregister_ctx]() { + mock_image_watcher.unregister_watch(&unregister_ctx); + }); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -EBLACKLISTED); + + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterWatchPendingUnregister) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + // inject an unregister + C_SaferCond unregister_ctx; + expect_aio_watch(mock_image_ctx, -ESHUTDOWN, + [&mock_image_watcher, &unregister_ctx]() { + mock_image_watcher.unregister_watch(&unregister_ctx); + }); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -ESHUTDOWN); + + ASSERT_EQ(0, unregister_ctx.wait()); +} + +TEST_F(TestMockWatcher, ReregisterPendingUnregister) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid); + + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + expect_aio_unwatch(mock_image_ctx, 0); + + // inject an unregister + C_SaferCond unregister_ctx; + expect_aio_watch(mock_image_ctx, 0, + [&mock_image_watcher, &unregister_ctx]() { + mock_image_watcher.unregister_watch(&unregister_ctx); + }); + + expect_aio_unwatch(mock_image_ctx, 0); + + C_SaferCond register_ctx; + mock_image_watcher.register_watch(®ister_ctx); + ASSERT_EQ(0, register_ctx.wait()); + + assert(m_watch_ctx != nullptr); + m_watch_ctx->handle_error(0, -ESHUTDOWN); + + ASSERT_EQ(0, unregister_ctx.wait()); +} + +} // namespace librbd diff --git a/src/test/librbd/watcher/test_mock_RewatchRequest.cc b/src/test/librbd/watcher/test_mock_RewatchRequest.cc index cd4dd297765b..58069d0a201c 100644 --- a/src/test/librbd/watcher/test_mock_RewatchRequest.cc +++ b/src/test/librbd/watcher/test_mock_RewatchRequest.cc @@ -19,6 +19,7 @@ using ::testing::InSequence; using ::testing::Invoke; using ::testing::Return; using ::testing::WithArg; +using ::testing::WithArgs; struct TestMockWatcherRewatchRequest : public TestMockFixture { typedef RewatchRequest MockRewatchRequest; @@ -32,7 +33,8 @@ struct TestMockWatcherRewatchRequest : public TestMockFixture { mock_image_ctx.md_ctx)); EXPECT_CALL(mock_io_ctx, aio_watch(mock_image_ctx.header_oid, _, _, _)) - .WillOnce(DoAll(WithArg<1>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c) { + .WillOnce(DoAll(WithArgs<1, 2>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c, uint64_t *cookie) { + *cookie = 234; c->get(); mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) { mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r); @@ -93,6 +95,7 @@ TEST_F(TestMockWatcherRewatchRequest, Success) { req->send(); } ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(234U, m_watch_handle); } TEST_F(TestMockWatcherRewatchRequest, UnwatchError) { @@ -117,6 +120,7 @@ TEST_F(TestMockWatcherRewatchRequest, UnwatchError) { req->send(); } ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(234U, m_watch_handle); } TEST_F(TestMockWatcherRewatchRequest, WatchBlacklist) { @@ -141,6 +145,7 @@ TEST_F(TestMockWatcherRewatchRequest, WatchBlacklist) { req->send(); } ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + ASSERT_EQ(0U, m_watch_handle); } TEST_F(TestMockWatcherRewatchRequest, WatchDNE) { @@ -165,6 +170,7 @@ TEST_F(TestMockWatcherRewatchRequest, WatchDNE) { req->send(); } ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(0U, m_watch_handle); } TEST_F(TestMockWatcherRewatchRequest, WatchError) { @@ -189,6 +195,34 @@ TEST_F(TestMockWatcherRewatchRequest, WatchError) { req->send(); } ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(0U, m_watch_handle); +} + +TEST_F(TestMockWatcherRewatchRequest, InvalidWatchHandler) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + + InSequence seq; + expect_aio_watch(mock_image_ctx, 0); + + m_watch_handle = 0; + + C_SaferCond ctx; + MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx.md_ctx, + mock_image_ctx.header_oid, + m_watch_lock, + &m_watch_ctx, + &m_watch_handle, + &ctx); + { + RWLock::WLocker watch_locker(m_watch_lock); + req->send(); + } + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(234U, m_watch_handle); } } // namespace watcher