From 7c45b13e2bad9101487fa2f86764ba8f31410b76 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 22 Nov 2016 15:11:02 -0500 Subject: [PATCH] rbd-mirror: async mirror image refresh state machine Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/CMakeLists.txt | 1 + .../test_mock_RefreshImagesRequest.cc | 151 ++++++++++++++++++ src/test/rbd_mirror/test_PoolWatcher.cc | 8 +- src/tools/rbd_mirror/CMakeLists.txt | 3 +- src/tools/rbd_mirror/PoolWatcher.cc | 4 +- src/tools/rbd_mirror/PoolWatcher.h | 19 --- src/tools/rbd_mirror/Replayer.cc | 2 +- src/tools/rbd_mirror/Replayer.h | 3 - .../pool_watcher/RefreshImagesRequest.cc | 139 ++++++++++++++++ .../pool_watcher/RefreshImagesRequest.h | 85 ++++++++++ src/tools/rbd_mirror/types.h | 24 +++ 11 files changed, 410 insertions(+), 29 deletions(-) create mode 100644 src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc create mode 100644 src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc create mode 100644 src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 259b2db6fa33..b815780650a3 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(unittest_rbd_mirror image_sync/test_mock_SnapshotCreateRequest.cc image_sync/test_mock_SyncPointCreateRequest.cc image_sync/test_mock_SyncPointPruneRequest.cc + pool_watcher/test_mock_RefreshImagesRequest.cc ) add_ceph_unittest(unittest_rbd_mirror ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_rbd_mirror) set_target_properties(unittest_rbd_mirror PROPERTIES COMPILE_FLAGS diff --git a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc new file mode 100644 index 000000000000..4e29dba97f60 --- /dev/null +++ b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc" +template class rbd::mirror::pool_watcher::RefreshImagesRequest; + +namespace rbd { +namespace mirror { +namespace pool_watcher { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockPoolWatcherRefreshImagesRequest : public TestMockFixture { +public: + typedef RefreshImagesRequest MockRefreshImagesRequest; + + void expect_mirror_image_list(librados::IoCtx &io_ctx, + const std::map &ids, + int r) { + bufferlist bl; + ::encode(ids, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_list"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_dir_list(librados::IoCtx &io_ctx, + const std::map &ids, int r) { + bufferlist bl; + ::encode(ids, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_DIRECTORY, _, StrEq("rbd"), StrEq("dir_list"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } +}; + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, Success) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + expect_dir_list(m_remote_io_ctx, {{"image name", "local id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ImageIds expected_image_ids = {{"global id", "local id", + boost::optional{"image name"}}}; + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, LargeDirectory) { + InSequence seq; + std::map mirror_list; + std::map dir_list; + ImageIds expected_image_ids; + for (uint32_t idx = 1; idx <= 1024; ++idx) { + mirror_list.insert(std::make_pair("local id " + stringify(idx), + "global id " + stringify(idx))); + dir_list.insert(std::make_pair("image " + stringify(idx), + "local id " + stringify(idx))); + expected_image_ids.insert({{"global id " + stringify(idx), + "local id " + stringify(idx), + "image " + stringify(idx)}}); + } + + expect_mirror_image_list(m_remote_io_ctx, mirror_list, 0); + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + expect_dir_list(m_remote_io_ctx, dir_list, 0); + expect_dir_list(m_remote_io_ctx, {{"image name", "local id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + expected_image_ids.insert({"global id", "local id", + boost::optional{"image name"}}); + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, MirrorImageListError) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {}, -EINVAL); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, DirListError) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + expect_dir_list(m_remote_io_ctx, {{"image name", "local id"}}, -EINVAL); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace pool_watcher +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc index beb95d1db3a5..433873b8586b 100644 --- a/src/test/rbd_mirror/test_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -26,6 +26,8 @@ #include #include +using rbd::mirror::ImageId; +using rbd::mirror::ImageIds; using rbd::mirror::PoolWatcher; using rbd::mirror::peer_t; using rbd::mirror::RadosRef; @@ -108,7 +110,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), sizeof(mirror_image_info))); image.close(); - m_mirrored_images.insert(PoolWatcher::ImageId( + m_mirrored_images.insert(ImageId( mirror_image_info.global_id, get_image_id(&ioctx, name), name)); } if (image_name != nullptr) @@ -154,7 +156,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), sizeof(mirror_image_info))); image.close(); - m_mirrored_images.insert(PoolWatcher::ImageId( + m_mirrored_images.insert(ImageId( mirror_image_info.global_id, get_image_id(&cioctx, name), name)); } if (image_name != nullptr) @@ -173,7 +175,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), unique_ptr m_pool_watcher; set m_pools; - PoolWatcher::ImageIds m_mirrored_images; + ImageIds m_mirrored_images; uint64_t m_image_number; uint64_t m_snap_number; diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index faa10b3523bd..10d6c8918d5d 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -27,7 +27,8 @@ set(rbd_mirror_internal image_sync/SnapshotCopyRequest.cc image_sync/SnapshotCreateRequest.cc image_sync/SyncPointCreateRequest.cc - image_sync/SyncPointPruneRequest.cc) + image_sync/SyncPointPruneRequest.cc + pool_watcher/RefreshImagesRequest.cc) add_library(rbd_mirror_internal STATIC ${rbd_mirror_internal}) diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 23b2cca17bdd..92c60699d56f 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -54,7 +54,7 @@ bool PoolWatcher::is_blacklisted() const { return m_blacklisted; } -const PoolWatcher::ImageIds& PoolWatcher::get_images() const +const ImageIds& PoolWatcher::get_images() const { assert(m_lock.is_locked()); return m_images; @@ -124,7 +124,7 @@ int PoolWatcher::refresh(ImageIds *image_ids) { return r; } for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { - boost::optional image_name(boost::none); + std::string image_name; auto it2 = image_id_to_name.find(it->first); if (it2 != image_id_to_name.end()) { image_name = it2->second; diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index 9e1e75b64af3..721106f761ed 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -25,25 +25,6 @@ namespace mirror { */ class PoolWatcher { public: - struct ImageId { - std::string global_id; - std::string id; - boost::optional name; - - ImageId(const std::string &global_id, const std::string &id = "", - const boost::optional &name = boost::none) - : global_id(global_id), id(id), name(name) { - } - - inline bool operator==(const ImageId &rhs) const { - return (global_id == rhs.global_id && id == rhs.id && name == rhs.name); - } - inline bool operator<(const ImageId &rhs) const { - return global_id < rhs.global_id; - } - }; - typedef std::set ImageIds; - PoolWatcher(librados::IoCtx &remote_io_ctx, double interval_seconds, Mutex &lock, Cond &cond); ~PoolWatcher(); diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index e067e19b0cef..d20128c5076b 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -605,7 +605,7 @@ void Replayer::set_sources(const ImageIds &image_ids) // shut down replayers for non-mirrored images for (auto image_it = m_image_replayers.begin(); image_it != m_image_replayers.end();) { - auto image_id_it = image_ids.find(image_it->first); + auto image_id_it = image_ids.find(ImageId(image_it->first)); if (image_id_it == image_ids.end()) { if (image_it->second->is_running()) { dout(20) << "stop image replayer for remote image " diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h index c60c86c88341..42db5ec964db 100644 --- a/src/tools/rbd_mirror/Replayer.h +++ b/src/tools/rbd_mirror/Replayer.h @@ -55,9 +55,6 @@ public: void release_leader(); private: - typedef PoolWatcher::ImageId ImageId; - typedef PoolWatcher::ImageIds ImageIds; - void init_local_mirroring_images(); void set_sources(const ImageIds &image_ids); diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc b/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc new file mode 100644 index 000000000000..5a2084aad007 --- /dev/null +++ b/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc @@ -0,0 +1,139 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/Utils.h" +#include + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::pool_watcher::RefreshImagesRequest " \ + << this << " " << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace pool_watcher { + +static const uint32_t MAX_RETURN = 1024; + +using librbd::util::create_rados_ack_callback; + +template +void RefreshImagesRequest::send() { + mirror_image_list(); +} + +template +void RefreshImagesRequest::mirror_image_list() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_list_start(&op, m_start_after, MAX_RETURN); + + librados::AioCompletion *aio_comp = create_rados_ack_callback< + RefreshImagesRequest, + &RefreshImagesRequest::handle_mirror_image_list>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void RefreshImagesRequest::handle_mirror_image_list(int r) { + dout(10) << "r=" << r << dendl; + + std::map ids; + if (r == 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_list_finish(&it, &ids); + } + + if (r < 0) { + derr << "failed to list mirrored images: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (!ids.empty()) { + m_local_to_global_ids.insert(ids.begin(), ids.end()); + if (ids.size() == MAX_RETURN) { + m_start_after = ids.rbegin()->first; + mirror_image_list(); + return; + } + } + + dir_list(); +} + +template +void RefreshImagesRequest::dir_list() { + dout(10) << dendl; + + m_out_bl.clear(); + m_start_after = ""; + + librados::ObjectReadOperation op; + librbd::cls_client::dir_list_start(&op, m_start_after, MAX_RETURN); + + librados::AioCompletion *aio_comp = create_rados_ack_callback< + RefreshImagesRequest, + &RefreshImagesRequest::handle_dir_list>(this); + int r = m_remote_io_ctx.aio_operate(RBD_DIRECTORY, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void RefreshImagesRequest::handle_dir_list(int r) { + dout(10) << "r=" << r << dendl; + + std::map name_to_ids; + if (r == 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = librbd::cls_client::dir_list_finish(&it, &name_to_ids); + } + + if (r < 0) { + derr << "failed to list images: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (!name_to_ids.empty()) { + for (auto &pair : name_to_ids) { + auto it = m_local_to_global_ids.find(pair.second); + if (it != m_local_to_global_ids.end()) { + // mirrored image must exist within directory to be treated as + // a valid image + m_image_ids->insert(ImageId(it->second, it->first, pair.first)); + } + } + + if (name_to_ids.size() == MAX_RETURN) { + m_start_after = name_to_ids.rbegin()->first; + dir_list(); + return; + } + } + + finish(0); +} + +template +void RefreshImagesRequest::finish(int r) { + dout(10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace pool_watcher +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::pool_watcher::RefreshImagesRequest; diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h b/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h new file mode 100644 index 000000000000..38dd1f5d53bb --- /dev/null +++ b/src/tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h @@ -0,0 +1,85 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_POOL_WATCHER_REFRESH_IMAGES_REQUEST_H +#define CEPH_RBD_MIRROR_POOL_WATCHER_REFRESH_IMAGES_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "tools/rbd_mirror/types.h" +#include +#include + +struct Context; + +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace pool_watcher { + +template +class RefreshImagesRequest { +public: + RefreshImagesRequest *create(librados::IoCtx &remote_io_ctx, + ImageIds *image_ids, Context *on_finish) { + return new RefreshImagesRequest(remote_io_ctx, image_ids, on_finish); + } + + RefreshImagesRequest(librados::IoCtx &remote_io_ctx, ImageIds *image_ids, + Context *on_finish) + : m_remote_io_ctx(remote_io_ctx), m_image_ids(image_ids), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | /-------------\ + * | | | + * v v | (more images) + * MIRROR_IMAGE_LIST ---/ + * | + * | /-------------\ + * | | | + * v v | (more images) + * DIR_LIST ------------/ + * | + * v + * + * + * @endverbatim + */ + + typedef std::unordered_map LocalToGlobalIds; + + librados::IoCtx &m_remote_io_ctx; + ImageIds *m_image_ids; + Context *m_on_finish; + + bufferlist m_out_bl; + std::string m_start_after; + LocalToGlobalIds m_local_to_global_ids; + + void mirror_image_list(); + void handle_mirror_image_list(int r); + + void dir_list(); + void handle_dir_list(int r); + + void finish(int r); + +}; + +} // namespace pool_watcher +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::pool_watcher::RefreshImagesRequest; + +#endif // CEPH_RBD_MIRROR_POOL_WATCHER_REFRESH_IMAGES_REQUEST_H diff --git a/src/tools/rbd_mirror/types.h b/src/tools/rbd_mirror/types.h index a1181ed19259..0a13049f5813 100644 --- a/src/tools/rbd_mirror/types.h +++ b/src/tools/rbd_mirror/types.h @@ -6,8 +6,10 @@ #include #include +#include #include #include +#include #include "include/rbd/librbd.hpp" #include "ImageSyncThrottler.h" @@ -22,6 +24,28 @@ typedef shared_ptr ImageRef; template using ImageSyncThrottlerRef = std::shared_ptr>; +struct ImageId { + std::string global_id; + std::string id; + boost::optional name; + + explicit ImageId(const std::string &global_id) : global_id(global_id) { + } + ImageId(const std::string &global_id, const std::string &id, + const boost::optional &name = boost::none) + : global_id(global_id), id(id), name(name) { + } + + inline bool operator==(const ImageId &rhs) const { + return (global_id == rhs.global_id && id == rhs.id && name == rhs.name); + } + inline bool operator<(const ImageId &rhs) const { + return global_id < rhs.global_id; + } +}; + +typedef std::set ImageIds; + struct peer_t { peer_t() = default; peer_t(const std::string &uuid, const std::string &cluster_name, -- 2.47.3