From 8d3e0e093e8cd532acb090c4585e2dfdfb36a6a6 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Thu, 14 Dec 2017 18:52:12 +0200 Subject: [PATCH] librbd: async request to list watchers Signed-off-by: Mykola Golub --- src/librbd/CMakeLists.txt | 1 + src/librbd/image/ListWatchersRequest.cc | 184 +++++++++++++++ src/librbd/image/ListWatchersRequest.h | 87 +++++++ src/test/librbd/CMakeLists.txt | 1 + .../image/test_mock_ListWatchersRequest.cc | 220 ++++++++++++++++++ 5 files changed, 493 insertions(+) create mode 100644 src/librbd/image/ListWatchersRequest.cc create mode 100644 src/librbd/image/ListWatchersRequest.h create mode 100644 src/test/librbd/image/test_mock_ListWatchersRequest.cc diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index b9c08d4624fa7..2cd5779f7da32 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -49,6 +49,7 @@ set(librbd_internal_srcs image/CloseRequest.cc image/CreateRequest.cc image/DetachChildRequest.cc + image/ListWatchersRequest.cc image/OpenRequest.cc image/RefreshParentRequest.cc image/RefreshRequest.cc diff --git a/src/librbd/image/ListWatchersRequest.cc b/src/librbd/image/ListWatchersRequest.cc new file mode 100644 index 0000000000000..21e8bfa454894 --- /dev/null +++ b/src/librbd/image/ListWatchersRequest.cc @@ -0,0 +1,184 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "ListWatchersRequest.h" +#include "common/RWLock.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageWatcher.h" +#include "librbd/Utils.h" + +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::ListWatchersRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace image { + +using librados::IoCtx; +using util::create_rados_callback; + +template +ListWatchersRequest::ListWatchersRequest(I &image_ctx, int flags, + std::list *watchers, + Context *on_finish) + : m_image_ctx(image_ctx), m_flags(flags), m_watchers(watchers), + m_on_finish(on_finish), m_cct(m_image_ctx.cct) { +} + +template +void ListWatchersRequest::send() { + ldout(m_cct, 20) << dendl; + + list_image_watchers(); +} + +template +void ListWatchersRequest::list_image_watchers() { + ldout(m_cct, 20) << dendl; + + librados::ObjectReadOperation op; + op.list_watchers(&m_object_watchers, &m_ret_val); + + using klass = ListWatchersRequest; + librados::AioCompletion *rados_completion = + create_rados_callback(this); + + int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid, + rados_completion, &op, &m_out_bl); + assert(r == 0); + rados_completion->release(); +} + +template +void ListWatchersRequest::handle_list_image_watchers(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r == 0 && m_ret_val < 0) { + r = m_ret_val; + } + if (r < 0) { + lderr(m_cct) << "error listing image watchers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + get_mirror_image(); +} + +template +void ListWatchersRequest::get_mirror_image() { + ldout(m_cct, 20) << dendl; + if ((m_object_watchers.empty()) || + (m_flags & LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES) == 0 || + (m_image_ctx.features & RBD_FEATURE_JOURNALING) == 0) { + finish(0); + return; + } + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_ctx.id); + + using klass = ListWatchersRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + assert(r == 0); + comp->release(); +} + +template +void ListWatchersRequest::handle_get_mirror_image(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r == -ENOENT) { + finish(0); + return; + } else if (r < 0) { + ldout(m_cct, 1) << "error retrieving mirror image: " << cpp_strerror(r) + << dendl; + } + + list_mirror_watchers(); +} + +template +void ListWatchersRequest::list_mirror_watchers() { + ldout(m_cct, 20) << dendl; + + librados::ObjectReadOperation op; + op.list_watchers(&m_mirror_watchers, &m_ret_val); + + using klass = ListWatchersRequest; + librados::AioCompletion *rados_completion = + create_rados_callback(this); + m_out_bl.clear(); + int r = m_image_ctx.md_ctx.aio_operate(RBD_MIRRORING, rados_completion, + &op, &m_out_bl); + assert(r == 0); + rados_completion->release(); +} + +template +void ListWatchersRequest::handle_list_mirror_watchers(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r == 0 && m_ret_val < 0) { + r = m_ret_val; + } + if (r < 0 && r != -ENOENT) { + ldout(m_cct, 1) << "error listing mirror watchers: " << cpp_strerror(r) + << dendl; + } + finish(0); +} + +template +void ListWatchersRequest::finish(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r == 0) { + m_watchers->clear(); + + if (m_object_watchers.size() > 0) { + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + uint64_t watch_handle = m_image_ctx.image_watcher != nullptr ? + m_image_ctx.image_watcher->get_watch_handle() : 0; + + for (auto &w : m_object_watchers) { + if ((m_flags & LIST_WATCHERS_FILTER_OUT_MY_INSTANCE) != 0) { + if (w.cookie == watch_handle) { + continue; + } + } + if ((m_flags & LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES) != 0) { + auto it = std::find_if(m_mirror_watchers.begin(), + m_mirror_watchers.end(), + [w] (obj_watch_t &watcher) { + return (strncmp(w.addr, watcher.addr, + sizeof(w.addr)) == 0); + }); + if (it != m_mirror_watchers.end()) { + continue; + } + } + m_watchers->push_back(w); + } + } + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::ListWatchersRequest; diff --git a/src/librbd/image/ListWatchersRequest.h b/src/librbd/image/ListWatchersRequest.h new file mode 100644 index 0000000000000..660098183bcca --- /dev/null +++ b/src/librbd/image/ListWatchersRequest.h @@ -0,0 +1,87 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_LIST_WATCHERS_REQUEST_H +#define CEPH_LIBRBD_IMAGE_LIST_WATCHERS_REQUEST_H + +#include "include/rados/rados_types.hpp" + +#include + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace image { + +enum { + LIST_WATCHERS_FILTER_OUT_MY_INSTANCE = 1 << 0, + LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES = 1 << 1, +}; + +template +class ListWatchersRequest { +public: + static ListWatchersRequest *create(ImageCtxT &image_ctx, int flags, + std::list *watchers, + Context *on_finish) { + return new ListWatchersRequest(image_ctx, flags, watchers, on_finish); + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * LIST_IMAGE_WATCHERS + * | + * v + * GET_MIRROR_IMAGE (skip if not needed) + * | + * v + * LIST_MIRROR_WATCHERS (skip if not needed) + * | + * v + * + * + * @endverbatim + */ + + ListWatchersRequest(ImageCtxT &image_ctx, int flags, std::list *watchers, + Context *on_finish); + + ImageCtxT& m_image_ctx; + int m_flags; + std::list *m_watchers; + Context *m_on_finish; + + CephContext *m_cct; + int m_ret_val; + bufferlist m_out_bl; + std::list m_object_watchers; + std::list m_mirror_watchers; + + void list_image_watchers(); + void handle_list_image_watchers(int r); + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void list_mirror_watchers(); + void handle_list_mirror_watchers(int r); + + void finish(int r); +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::ListWatchersRequest; + +#endif // CEPH_LIBRBD_IMAGE_LIST_WATCHERS_REQUEST_H diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 35b9d69802be6..ced4f678e3ce6 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -46,6 +46,7 @@ set(unittest_librbd_srcs exclusive_lock/test_mock_PreReleaseRequest.cc image/test_mock_CloneRequest.cc image/test_mock_DetachChildRequest.cc + image/test_mock_ListWatchersRequest.cc image/test_mock_RefreshRequest.cc image/test_mock_RemoveRequest.cc io/test_mock_ImageRequest.cc diff --git a/src/test/librbd/image/test_mock_ListWatchersRequest.cc b/src/test/librbd/image/test_mock_ListWatchersRequest.cc new file mode 100644 index 0000000000000..8090379b48074 --- /dev/null +++ b/src/test/librbd/image/test_mock_ListWatchersRequest.cc @@ -0,0 +1,220 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "librbd/image/ListWatchersRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockContextWQ.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/test_support.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "librbd/image/ListWatchersRequest.cc" +template class librbd::image::ListWatchersRequest; + +namespace librbd { + +namespace image { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; + +class TestMockListWatchersRequest : public TestMockFixture { +public: + typedef ListWatchersRequest MockListWatchersRequest; + + obj_watch_t watcher(const std::string &address, uint64_t watch_handle) { + obj_watch_t w; + strcpy(w.addr, address.c_str()); + w.watcher_id = 0; + w.cookie = watch_handle; + w.timeout_seconds = 0; + + return w; + } + + void expect_list_watchers(MockTestImageCtx &mock_image_ctx, + const std::string oid, + const std::list &watchers, int r) { + auto &expect = EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + list_watchers(oid, _)); + if (r < 0) { + expect.WillOnce(Return(r)); + } else { + expect.WillOnce(DoAll(SetArgPointee<1>(watchers), Return(0))); + } + } + + void expect_list_image_watchers(MockTestImageCtx &mock_image_ctx, + const std::list &watchers, + int r) { + expect_list_watchers(mock_image_ctx, mock_image_ctx.header_oid, + watchers, r); + } + + void expect_list_mirror_watchers(MockTestImageCtx &mock_image_ctx, + const std::list &watchers, + int r) { + expect_list_watchers(mock_image_ctx, RBD_MIRRORING, watchers, r); + } + + void expect_mirror_image_get(MockImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get"), + _, _, _)) + .WillOnce(Return(r)); + } + + void expect_get_watch_handle(MockImageWatcher &mock_watcher, + uint64_t watch_handle) { + EXPECT_CALL(mock_watcher, get_watch_handle()) + .WillOnce(Return(watch_handle)); + } +}; + +TEST_F(TestMockListWatchersRequest, NoImageWatchers) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockImageWatcher mock_watcher; + + InSequence seq; + expect_list_image_watchers(mock_image_ctx, {}, 0); + + std::list watchers; + C_SaferCond ctx; + auto req = MockListWatchersRequest::create(mock_image_ctx, 0, &watchers, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(watchers.empty()); +} + +TEST_F(TestMockListWatchersRequest, Error) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockImageWatcher mock_watcher; + + InSequence seq; + expect_list_image_watchers(mock_image_ctx, {}, -EINVAL); + + std::list watchers; + C_SaferCond ctx; + auto req = MockListWatchersRequest::create(mock_image_ctx, 0, &watchers, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockListWatchersRequest, Success) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockImageWatcher mock_watcher; + + InSequence seq; + expect_list_image_watchers(mock_image_ctx, + {watcher("a", 123), watcher("b", 456)}, 0); + expect_get_watch_handle(*mock_image_ctx.image_watcher, 123); + + std::list watchers; + C_SaferCond ctx; + auto req = MockListWatchersRequest::create(mock_image_ctx, 0, &watchers, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(2U, watchers.size()); + + auto w = watchers.begin(); + ASSERT_STREQ("a", w->addr); + ASSERT_EQ(123, w->cookie); + + w++; + ASSERT_STREQ("b", w->addr); + ASSERT_EQ(456, w->cookie); +} + +TEST_F(TestMockListWatchersRequest, FilterOutMyInstance) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockImageWatcher mock_watcher; + + InSequence seq; + expect_list_image_watchers(mock_image_ctx, + {watcher("a", 123), watcher("b", 456)}, 0); + expect_get_watch_handle(*mock_image_ctx.image_watcher, 123); + + std::list watchers; + C_SaferCond ctx; + auto req = MockListWatchersRequest::create( + mock_image_ctx, LIST_WATCHERS_FILTER_OUT_MY_INSTANCE, &watchers, &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(1U, watchers.size()); + + ASSERT_STREQ("b", watchers.begin()->addr); + ASSERT_EQ(456, watchers.begin()->cookie); +} + +TEST_F(TestMockListWatchersRequest, FilterOutMirrorInstance) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockImageWatcher mock_watcher; + + InSequence seq; + expect_list_image_watchers(mock_image_ctx, + {watcher("a", 123), watcher("b", 456)}, 0); + expect_mirror_image_get(mock_image_ctx, 0); + expect_list_mirror_watchers(mock_image_ctx, {watcher("b", 789)}, 0); + expect_get_watch_handle(*mock_image_ctx.image_watcher, 123); + + std::list watchers; + C_SaferCond ctx; + auto req = MockListWatchersRequest::create( + mock_image_ctx, LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES, &watchers, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(1U, watchers.size()); + + ASSERT_STREQ("a", watchers.begin()->addr); + ASSERT_EQ(123, watchers.begin()->cookie); +} + +} // namespace image +} // namespace librbd -- 2.39.5