From 3b49b57d56281be9b3b7f5e079f805917ba0f735 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Sun, 30 Sep 2018 20:14:54 +0300 Subject: [PATCH] rbd-mirror: use pool level config overrides Signed-off-by: Mykola Golub --- .../test_mock_PrepareRemoteImageRequest.cc | 12 ++++++------ src/test/rbd_mirror/test_ImageReplayer.cc | 10 ++-------- src/test/rbd_mirror/test_mock_ImageReplayer.cc | 1 + .../rbd_mirror/test_mock_ImageSyncThrottler.cc | 14 +++++++------- src/test/rbd_mirror/test_mock_InstanceWatcher.cc | 2 +- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 14 ++++++++++++++ src/tools/rbd_mirror/ImageDeleter.cc | 6 ++++-- src/tools/rbd_mirror/ImageReplayer.cc | 12 ++++++++++-- src/tools/rbd_mirror/ImageSyncThrottler.cc | 11 ++++++----- src/tools/rbd_mirror/ImageSyncThrottler.h | 8 +++++--- src/tools/rbd_mirror/InstanceReplayer.cc | 3 ++- src/tools/rbd_mirror/InstanceWatcher.cc | 2 +- src/tools/rbd_mirror/PoolReplayer.cc | 16 ++++++++++++++++ .../image_replayer/PrepareRemoteImageRequest.cc | 9 +-------- .../image_replayer/PrepareRemoteImageRequest.h | 12 +++++++++--- 15 files changed, 85 insertions(+), 47 deletions(-) diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc index 782702f0483..1b957ed177f 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -179,7 +179,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) { m_remote_io_ctx, "global image id", "local mirror uuid", - "local image id", + "local image id", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, @@ -227,7 +227,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { m_remote_io_ctx, "global image id", "local mirror uuid", - "local image id", + "local image id", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, @@ -260,7 +260,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) { m_remote_io_ctx, "global image id", "local mirror uuid", - "", + "", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, @@ -292,7 +292,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { m_remote_io_ctx, "global image id", "local mirror uuid", - "", + "", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, @@ -331,7 +331,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { m_remote_io_ctx, "global image id", "local mirror uuid", - "local image id", + "local image id", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, @@ -377,7 +377,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { m_remote_io_ctx, "global image id", "local mirror uuid", - "local image id", + "local image id", {}, &remote_mirror_uuid, &remote_image_id, &remote_journaler, diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 122182d7e42..51eb8b560c6 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -82,15 +82,11 @@ public: TestImageReplayer() : m_local_cluster(new librados::Rados()), m_watch_handle(0) { - EXPECT_EQ(0, g_ceph_context->_conf.get_val("rbd_mirror_journal_commit_age", - &m_journal_commit_age)); - EXPECT_EQ(0, g_ceph_context->_conf.set_val("rbd_mirror_journal_commit_age", - "0.1")); - EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get())); EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false")); EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1")); - + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_commit_age", + "0.1")); m_local_pool_name = get_temp_pool_name(); EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str())); EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(), @@ -144,8 +140,6 @@ public: EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); - EXPECT_EQ(0, g_ceph_context->_conf.set_val("rbd_mirror_journal_commit_age", - m_journal_commit_age)); } template > diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 85d22ec70a9..d9ed376a788 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -165,6 +165,7 @@ struct PrepareRemoteImageRequest { const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &local_image_id, + const journal::Settings &settings, std::string *remote_mirror_uuid, std::string *remote_image_id, ::journal::MockJournalerProxy **remote_journaler, diff --git a/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc b/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc index 24815aeb799..f30a299f150 100644 --- a/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc +++ b/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc @@ -42,7 +42,7 @@ public: }; TEST_F(TestMockImageSyncThrottler, Single_Sync) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); C_SaferCond on_start; throttler.start_op("id", &on_start); ASSERT_EQ(0, on_start.wait()); @@ -50,7 +50,7 @@ TEST_F(TestMockImageSyncThrottler, Single_Sync) { } TEST_F(TestMockImageSyncThrottler, Multiple_Syncs) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); throttler.set_max_concurrent_syncs(2); C_SaferCond on_start1; @@ -73,7 +73,7 @@ TEST_F(TestMockImageSyncThrottler, Multiple_Syncs) { } TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); C_SaferCond on_start; throttler.start_op("id", &on_start); ASSERT_EQ(0, on_start.wait()); @@ -82,7 +82,7 @@ TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync) { } TEST_F(TestMockImageSyncThrottler, Cancel_Waiting_Sync) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); throttler.set_max_concurrent_syncs(1); C_SaferCond on_start1; @@ -98,7 +98,7 @@ TEST_F(TestMockImageSyncThrottler, Cancel_Waiting_Sync) { TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync_Start_Waiting) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); throttler.set_max_concurrent_syncs(1); C_SaferCond on_start1; @@ -114,7 +114,7 @@ TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync_Start_Waiting) { } TEST_F(TestMockImageSyncThrottler, Increase_Max_Concurrent_Syncs) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); throttler.set_max_concurrent_syncs(2); C_SaferCond on_start1; @@ -146,7 +146,7 @@ TEST_F(TestMockImageSyncThrottler, Increase_Max_Concurrent_Syncs) { } TEST_F(TestMockImageSyncThrottler, Decrease_Max_Concurrent_Syncs) { - MockImageSyncThrottler throttler; + MockImageSyncThrottler throttler(g_ceph_context); throttler.set_max_concurrent_syncs(4); C_SaferCond on_start1; diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc index ede548dc043..ec2b7057df7 100644 --- a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc +++ b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc @@ -87,7 +87,7 @@ template <> struct ImageSyncThrottler { static ImageSyncThrottler* s_instance; - static ImageSyncThrottler *create() { + static ImageSyncThrottler *create(CephContext *cct) { ceph_assert(s_instance != nullptr); return s_instance; } diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 0709201f310..c7f0909cd51 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -1,6 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "librbd/api/Config.h" #include "test/librbd/mock/MockImageCtx.h" #include "test/librados_test_stub/MockTestMemCluster.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" @@ -30,6 +31,19 @@ struct MockTestImageCtx : public MockImageCtx { } // anonymous namespace +namespace api { + +template <> +class Config { +public: + static int list(librados::IoCtx& io_ctx, + std::vector *options) { + return 0; + } +}; + +} + } // namespace librbd namespace rbd { diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index e8439f273ca..93ddf5cea1b 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -356,7 +356,8 @@ template void ImageDeleter::remove_images() { dout(10) << dendl; - uint64_t max_concurrent_deletions = g_ceph_context->_conf.get_val( + auto cct = reinterpret_cast(m_local_io_ctx.cct()); + uint64_t max_concurrent_deletions = cct->_conf.get_val( "rbd_mirror_concurrent_image_deletions"); Mutex::Locker locker(m_lock); @@ -414,7 +415,8 @@ void ImageDeleter::handle_remove_image(DeleteInfoRef delete_info, image_deleter::ERROR_RESULT_RETRY_IMMEDIATELY) { enqueue_failed_delete(&delete_info, r, m_busy_interval); } else { - double failed_interval = g_ceph_context->_conf.get_val( + auto cct = reinterpret_cast(m_local_io_ctx.cct()); + double failed_interval = cct->_conf.get_val( "rbd_mirror_delete_retry_interval"); enqueue_failed_delete(&delete_info, r, failed_interval); } diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 35650e5d3e4..2a9a26f40fa 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -433,12 +433,20 @@ void ImageReplayer::prepare_remote_image() { ceph_assert(!m_peers.empty()); m_remote_image = {*m_peers.begin()}; + auto cct = static_cast(m_local->cct()); + journal::Settings journal_settings; + journal_settings.commit_interval = cct->_conf.get_val( + "rbd_mirror_journal_commit_age"); + journal_settings.max_fetch_bytes = cct->_conf.get_val( + "rbd_mirror_journal_max_fetch_bytes"); + Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_prepare_remote_image>(this); auto req = PrepareRemoteImageRequest::create( m_threads, m_remote_image.io_ctx, m_global_image_id, m_local_mirror_uuid, - m_local_image_id, &m_remote_image.mirror_uuid, &m_remote_image.image_id, - &m_remote_journaler, &m_client_state, &m_client_meta, ctx); + m_local_image_id, journal_settings, &m_remote_image.mirror_uuid, + &m_remote_image.image_id, &m_remote_journaler, &m_client_state, + &m_client_meta, ctx); req->send(); } diff --git a/src/tools/rbd_mirror/ImageSyncThrottler.cc b/src/tools/rbd_mirror/ImageSyncThrottler.cc index a3fc8330fdc..c2e618bf48f 100644 --- a/src/tools/rbd_mirror/ImageSyncThrottler.cc +++ b/src/tools/rbd_mirror/ImageSyncThrottler.cc @@ -28,18 +28,19 @@ namespace rbd { namespace mirror { template -ImageSyncThrottler::ImageSyncThrottler() - : m_lock(librbd::util::unique_lock_name("rbd::mirror::ImageSyncThrottler", +ImageSyncThrottler::ImageSyncThrottler(CephContext *cct) + : m_cct(cct), + m_lock(librbd::util::unique_lock_name("rbd::mirror::ImageSyncThrottler", this)), - m_max_concurrent_syncs(g_ceph_context->_conf.get_val( + m_max_concurrent_syncs(cct->_conf.get_val( "rbd_mirror_concurrent_image_syncs")) { dout(20) << "max_concurrent_syncs=" << m_max_concurrent_syncs << dendl; - g_ceph_context->_conf.add_observer(this); + m_cct->_conf.add_observer(this); } template ImageSyncThrottler::~ImageSyncThrottler() { - g_ceph_context->_conf.remove_observer(this); + m_cct->_conf.remove_observer(this); Mutex::Locker locker(m_lock); ceph_assert(m_inflight_ops.empty()); diff --git a/src/tools/rbd_mirror/ImageSyncThrottler.h b/src/tools/rbd_mirror/ImageSyncThrottler.h index 8f965bf7b7e..8c8f754626a 100644 --- a/src/tools/rbd_mirror/ImageSyncThrottler.h +++ b/src/tools/rbd_mirror/ImageSyncThrottler.h @@ -13,6 +13,7 @@ #include "common/Mutex.h" #include "common/config_obs.h" +class CephContext; class Context; namespace ceph { class Formatter; } @@ -24,14 +25,14 @@ namespace mirror { template class ImageSyncThrottler : public md_config_obs_t { public: - static ImageSyncThrottler *create() { - return new ImageSyncThrottler(); + static ImageSyncThrottler *create(CephContext *cct) { + return new ImageSyncThrottler(cct); } void destroy() { delete this; } - ImageSyncThrottler(); + ImageSyncThrottler(CephContext *cct); ~ImageSyncThrottler() override; void set_max_concurrent_syncs(uint32_t max); @@ -43,6 +44,7 @@ public: void print_status(Formatter *f, std::stringstream *ss); private: + CephContext *m_cct; Mutex m_lock; uint32_t m_max_concurrent_syncs; std::list> m_queue; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index a2165965f5d..d3dfc557d70 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -488,7 +488,8 @@ void InstanceReplayer::schedule_image_state_check_task() { queue_start_image_replayers(); }); - int after = g_ceph_context->_conf.get_val( + auto cct = static_cast(m_local_rados->cct()); + int after = cct->_conf.get_val( "rbd_mirror_image_state_check_interval"); dout(10) << "scheduling image state check after " << after << " sec (task " diff --git a/src/tools/rbd_mirror/InstanceWatcher.cc b/src/tools/rbd_mirror/InstanceWatcher.cc index 9ba9d01056a..afecd4042f5 100644 --- a/src/tools/rbd_mirror/InstanceWatcher.cc +++ b/src/tools/rbd_mirror/InstanceWatcher.cc @@ -595,7 +595,7 @@ void InstanceWatcher::handle_acquire_leader() { Mutex::Locker locker(m_lock); ceph_assert(m_image_sync_throttler == nullptr); - m_image_sync_throttler = ImageSyncThrottler::create(); + m_image_sync_throttler = ImageSyncThrottler::create(m_cct); m_leader_instance_id = m_instance_id; unsuspend_notify_requests(); diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 2dbaf93f8f6..8d0ec509df1 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -16,6 +16,7 @@ #include "librbd/internal.h" #include "librbd/Utils.h" #include "librbd/Watcher.h" +#include "librbd/api/Config.h" #include "librbd/api/Mirror.h" #include "ImageMap.h" #include "InstanceReplayer.h" @@ -305,6 +306,21 @@ void PoolReplayer::init() return; } + std::vector options; + r = librbd::api::Config::list(m_local_io_ctx, &options); + if (r < 0) { + derr << "error listing local pool config overrides: " << cpp_strerror(r) + << dendl; + return; + } + auto cct = reinterpret_cast(m_local_io_ctx.cct()); + for (auto &option : options) { + if (option.source == RBD_CONFIG_SOURCE_POOL) { + r = cct->_conf.set_val(option.name.c_str(), option.value); + assert(r == 0); + } + } + std::string local_mirror_uuid; r = librbd::cls_client::mirror_uuid_get(&m_local_io_ctx, &local_mirror_uuid); diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index 3ebd5992d43..00c141e0bc1 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -8,7 +8,6 @@ #include "common/errno.h" #include "common/WorkQueue.h" #include "journal/Journaler.h" -#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "librbd/journal/Types.h" @@ -106,17 +105,11 @@ template void PrepareRemoteImageRequest::get_client() { dout(20) << dendl; - journal::Settings settings; - settings.commit_interval = g_ceph_context->_conf.get_val( - "rbd_mirror_journal_commit_age"); - settings.max_fetch_bytes = g_ceph_context->_conf.get_val( - "rbd_mirror_journal_max_fetch_bytes"); - ceph_assert(*m_remote_journaler == nullptr); *m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, m_remote_io_ctx, *m_remote_image_id, m_local_mirror_uuid, - settings); + m_journal_settings); Context *ctx = create_async_context_callback( m_threads->work_queue, create_context_callback< diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h index 5f278a8c65c..e0b4898cac3 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -6,10 +6,12 @@ #include "include/buffer.h" #include "cls/journal/cls_journal_types.h" +#include "journal/Settings.h" #include "librbd/journal/TypeTraits.h" #include namespace journal { class Journaler; } +namespace journal { class Settings; } namespace librados { struct IoCtx; } namespace librbd { struct ImageCtx; } namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } @@ -36,6 +38,7 @@ public: const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &local_image_id, + const journal::Settings &settings, std::string *remote_mirror_uuid, std::string *remote_image_id, Journaler **remote_journaler, @@ -44,9 +47,10 @@ public: Context *on_finish) { return new PrepareRemoteImageRequest(threads, remote_io_ctx, global_image_id, local_mirror_uuid, - local_image_id, remote_mirror_uuid, - remote_image_id, remote_journaler, - client_state, client_meta, on_finish); + local_image_id, settings, + remote_mirror_uuid, remote_image_id, + remote_journaler, client_state, + client_meta, on_finish); } PrepareRemoteImageRequest(Threads *threads, @@ -54,6 +58,7 @@ public: const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &local_image_id, + const journal::Settings &journal_settings, std::string *remote_mirror_uuid, std::string *remote_image_id, Journaler **remote_journaler, @@ -100,6 +105,7 @@ private: std::string m_global_image_id; std::string m_local_mirror_uuid; std::string m_local_image_id; + journal::Settings m_journal_settings; std::string *m_remote_mirror_uuid; std::string *m_remote_image_id; Journaler **m_remote_journaler; -- 2.39.5