From 54819a1aa95fe1be8b8ef44f8417e3fe82d9ec89 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 27 Jul 2020 15:31:09 -0400 Subject: [PATCH] librbd: initial config watcher implementation The config watcher will initially observe all "rbd_" configuration updates received from the MON that have not been locally overridden at the pool and/or image level. Signed-off-by: Jason Dillaman --- src/librbd/CMakeLists.txt | 1 + src/librbd/ConfigWatcher.cc | 116 +++++++++++++++++++++ src/librbd/ConfigWatcher.h | 47 +++++++++ src/test/librbd/CMakeLists.txt | 1 + src/test/librbd/mock/MockImageCtx.h | 1 + src/test/librbd/mock/MockImageState.h | 2 + src/test/librbd/test_mock_ConfigWatcher.cc | 100 ++++++++++++++++++ 7 files changed, 268 insertions(+) create mode 100644 src/librbd/ConfigWatcher.cc create mode 100644 src/librbd/ConfigWatcher.h create mode 100644 src/test/librbd/test_mock_ConfigWatcher.cc diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 5f9063042ad79..00fcc1d75719d 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -9,6 +9,7 @@ set(librbd_internal_srcs AsioEngine.cc AsyncObjectThrottle.cc AsyncRequest.cc + ConfigWatcher.cc DeepCopyRequest.cc ExclusiveLock.cc ImageCtx.cc diff --git a/src/librbd/ConfigWatcher.cc b/src/librbd/ConfigWatcher.cc new file mode 100644 index 0000000000000..0e412780438cf --- /dev/null +++ b/src/librbd/ConfigWatcher.cc @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/ConfigWatcher.h" +#include "common/config_obs.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/api/Config.h" +#include +#include +#include +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::ConfigWatcher: " \ + << __func__ << ": " + +namespace librbd { + +template +struct ConfigWatcher::Observer : public md_config_obs_t { + ConfigWatcher* m_config_watcher; + + std::deque m_config_key_strs; + mutable std::vector m_config_keys; + + Observer(CephContext* cct, ConfigWatcher* config_watcher) + : m_config_watcher(config_watcher) { + const std::string rbd_key_prefix("rbd_"); + auto& schema = cct->_conf.get_schema(); + for (auto& pair : schema) { + // watch all "rbd_" keys for simplicity + if (!boost::starts_with(pair.first, rbd_key_prefix)) { + continue; + } + + m_config_key_strs.emplace_back(pair.first); + } + + m_config_keys.reserve(m_config_key_strs.size()); + for (auto& key : m_config_key_strs) { + m_config_keys.emplace_back(key.c_str()); + } + m_config_keys.emplace_back(nullptr); + } + + const char** get_tracked_conf_keys() const override { + ceph_assert(!m_config_keys.empty()); + return &m_config_keys[0]; + } + + void handle_conf_change(const ConfigProxy& conf, + const std::set &changed) override { + m_config_watcher->handle_global_config_change(changed); + } +}; + +template +ConfigWatcher::ConfigWatcher(I& image_ctx) + : m_image_ctx(image_ctx) { +} + +template +ConfigWatcher::~ConfigWatcher() { + ceph_assert(m_observer == nullptr); +} + +template +void ConfigWatcher::init() { + auto cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + m_observer = new Observer(cct, this); + cct->_conf.add_observer(m_observer); +} + +template +void ConfigWatcher::shut_down() { + auto cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + ceph_assert(m_observer != nullptr); + cct->_conf.remove_observer(m_observer); + + delete m_observer; + m_observer = nullptr; +} + +template +void ConfigWatcher::handle_global_config_change( + std::set changed_keys) { + + { + // ignore any global changes that are being overridden + std::shared_lock image_locker{m_image_ctx.image_lock}; + for (auto& key : m_image_ctx.config_overrides) { + changed_keys.erase(key); + } + } + if (changed_keys.empty()) { + return; + } + + auto cct = m_image_ctx.cct; + ldout(cct, 10) << "changed_keys=" << changed_keys << dendl; + + // refresh the image to pick up any global config overrides + m_image_ctx.state->handle_update_notification(); +} + +} // namespace librbd + +template class librbd::ConfigWatcher; diff --git a/src/librbd/ConfigWatcher.h b/src/librbd/ConfigWatcher.h new file mode 100644 index 0000000000000..1f10c8cb87ebc --- /dev/null +++ b/src/librbd/ConfigWatcher.h @@ -0,0 +1,47 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_CONFIG_WATCHER_H +#define CEPH_LIBRBD_CONFIG_WATCHER_H + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +template +class ConfigWatcher { +public: + static ConfigWatcher* create(ImageCtxT& image_ctx) { + return new ConfigWatcher(image_ctx); + } + + ConfigWatcher(ImageCtxT& image_ctx); + ~ConfigWatcher(); + + ConfigWatcher(const ConfigWatcher&) = delete; + ConfigWatcher& operator=(const ConfigWatcher&) = delete; + + void init(); + void shut_down(); + +private: + struct Observer; + + ImageCtxT& m_image_ctx; + + Observer* m_observer = nullptr; + + void handle_global_config_change(std::set changed); + +}; + +} // namespace librbd + +extern template class librbd::ConfigWatcher; + +#endif // CEPH_LIBRBD_CONFIG_WATCHER_H diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 8189b46d75559..403c3c12ccdfe 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -43,6 +43,7 @@ target_link_libraries(rbd_test_mock PUBLIC set(unittest_librbd_srcs test_main.cc test_mock_fixture.cc + test_mock_ConfigWatcher.cc test_mock_DeepCopyRequest.cc test_mock_ExclusiveLock.cc test_mock_Journal.cc diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h index a06ce09e6a5dc..82acb2623b349 100644 --- a/src/test/librbd/mock/MockImageCtx.h +++ b/src/test/librbd/mock/MockImageCtx.h @@ -329,6 +329,7 @@ struct MockImageCtx { bool cache; ConfigProxy config; + std::set config_overrides; }; } // namespace librbd diff --git a/src/test/librbd/mock/MockImageState.h b/src/test/librbd/mock/MockImageState.h index 9b9dc8b879a55..f510c4a0fd835 100644 --- a/src/test/librbd/mock/MockImageState.h +++ b/src/test/librbd/mock/MockImageState.h @@ -30,6 +30,8 @@ struct MockImageState { MOCK_METHOD2(register_update_watcher, int(UpdateWatchCtx *, uint64_t *)); MOCK_METHOD2(unregister_update_watcher, void(uint64_t, Context *)); + + MOCK_METHOD0(handle_update_notification, void()); }; } // namespace librbd diff --git a/src/test/librbd/test_mock_ConfigWatcher.cc b/src/test/librbd/test_mock_ConfigWatcher.cc new file mode 100644 index 0000000000000..264e3d2c11ab0 --- /dev/null +++ b/src/test/librbd/test_mock_ConfigWatcher.cc @@ -0,0 +1,100 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "test/librbd/test_support.h" +#include "include/rbd_types.h" +#include "common/ceph_mutex.h" +#include "librbd/ConfigWatcher.h" +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include + +namespace librbd { +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(ImageCtx &image_ctx) : MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +#include "librbd/ConfigWatcher.cc" + +namespace librbd { + +using ::testing::Invoke; + +class TestMockConfigWatcher : public TestMockFixture { +public: + typedef ConfigWatcher MockConfigWatcher; + + librbd::ImageCtx *m_image_ctx; + + ceph::mutex m_lock = ceph::make_mutex("m_lock"); + ceph::condition_variable m_cv; + bool m_refreshed = false; + + void SetUp() override { + TestMockFixture::SetUp(); + + ASSERT_EQ(0, open_image(m_image_name, &m_image_ctx)); + } + + void expect_update_notification(MockTestImageCtx& mock_image_ctx) { + EXPECT_CALL(*mock_image_ctx.state, handle_update_notification()) + .WillOnce(Invoke([this]() { + std::unique_lock locker{m_lock}; + m_refreshed = true; + m_cv.notify_all(); + })); + } + + void wait_for_update_notification() { + std::unique_lock locker{m_lock}; + m_cv.wait(locker, [this] { + if (m_refreshed) { + m_refreshed = false; + return true; + } + return false; + }); + } +}; + +TEST_F(TestMockConfigWatcher, GlobalConfig) { + MockTestImageCtx mock_image_ctx(*m_image_ctx); + + MockConfigWatcher mock_config_watcher(mock_image_ctx); + mock_config_watcher.init(); + + expect_update_notification(mock_image_ctx); + mock_image_ctx.cct->_conf.set_val("rbd_cache", "false"); + mock_image_ctx.cct->_conf.set_val("rbd_cache", "true"); + mock_image_ctx.cct->_conf.apply_changes(nullptr); + wait_for_update_notification(); + + mock_config_watcher.shut_down(); +} + +TEST_F(TestMockConfigWatcher, IgnoreOverriddenGlobalConfig) { + MockTestImageCtx mock_image_ctx(*m_image_ctx); + + MockConfigWatcher mock_config_watcher(mock_image_ctx); + mock_config_watcher.init(); + + EXPECT_CALL(*mock_image_ctx.state, handle_update_notification()) + .Times(0); + mock_image_ctx.config_overrides.insert("rbd_cache"); + mock_image_ctx.cct->_conf.set_val("rbd_cache", "false"); + mock_image_ctx.cct->_conf.set_val("rbd_cache", "true"); + mock_image_ctx.cct->_conf.apply_changes(nullptr); + + mock_config_watcher.shut_down(); + + ASSERT_FALSE(m_refreshed); +} + +} // namespace librbd -- 2.39.5