From afb62232a4c0ca243d34ccde0f47e01ec6e07bab Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Thu, 14 Nov 2019 16:13:22 +0000 Subject: [PATCH] rbd-mirror: add namespace support to service daemon Signed-off-by: Mykola Golub --- .../rbd_mirror/test_mock_InstanceReplayer.cc | 7 +- .../rbd_mirror/test_mock_NamespaceReplayer.cc | 24 +----- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 49 +++++++++++- src/tools/rbd_mirror/InstanceReplayer.cc | 18 ++--- src/tools/rbd_mirror/NamespaceReplayer.cc | 26 ++----- src/tools/rbd_mirror/PoolReplayer.cc | 8 ++ src/tools/rbd_mirror/ServiceDaemon.cc | 77 +++++++++++++++++++ src/tools/rbd_mirror/ServiceDaemon.h | 8 ++ 8 files changed, 164 insertions(+), 53 deletions(-) diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index 0d1b620c53f..743c709bdda 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -50,8 +50,8 @@ struct Threads { template<> struct ServiceDaemon { - MOCK_METHOD3(add_or_update_attribute, - void(int64_t, const std::string&, + MOCK_METHOD4(add_or_update_namespace_attribute, + void(int64_t, const std::string&, const std::string&, const service_daemon::AttributeValue&)); }; @@ -299,7 +299,8 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) { EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, destroy()); - EXPECT_CALL(mock_service_daemon,add_or_update_attribute(_, _, _)).Times(3); + EXPECT_CALL(mock_service_daemon, + add_or_update_namespace_attribute(_, _, _, _)).Times(3); ASSERT_TRUE(start_image_replayers_ctx != nullptr); start_image_replayers_ctx->complete(0); diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index 6cde475a2f7..162d2abef02 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -221,8 +221,8 @@ std::map *> PoolWatcher struct ServiceDaemon { - MOCK_METHOD3(add_or_update_attribute, - void(int64_t, const std::string&, + MOCK_METHOD4(add_or_update_namespace_attribute, + void(int64_t, const std::string&, const std::string&, const service_daemon::AttributeValue&)); MOCK_METHOD2(remove_attribute, void(int64_t, const std::string&)); @@ -386,20 +386,6 @@ public: .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); } - void expect_service_daemon_add_or_update_attribute( - MockServiceDaemon &mock_service_daemon, const std::string& key, - const service_daemon::AttributeValue& value) { - EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, key, value)); - } - - void expect_service_daemon_add_or_update_instance_id_attribute( - MockInstanceWatcher &mock_instance_watcher, - MockServiceDaemon &mock_service_daemon) { - expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234"); - expect_service_daemon_add_or_update_attribute( - mock_service_daemon, "instance_id", {std::string("1234")}); - } - MockThreads *m_mock_threads; }; @@ -515,9 +501,6 @@ TEST_F(TestMockNamespaceReplayer, Init) { expect_instance_watcher_init(*mock_instance_watcher, 0); MockServiceDaemon mock_service_daemon; - expect_service_daemon_add_or_update_instance_id_attribute( - *mock_instance_watcher, mock_service_daemon); - MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, @@ -558,9 +541,6 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) { expect_instance_watcher_init(*mock_instance_watcher, 0); MockServiceDaemon mock_service_daemon; - expect_service_daemon_add_or_update_instance_id_attribute( - *mock_instance_watcher, mock_service_daemon); - MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 039ab7292e4..756d3906490 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -202,6 +202,9 @@ LeaderWatcher* LeaderWatcher template<> struct ServiceDaemon { + MOCK_METHOD2(add_namespace, void(int64_t, const std::string &)); + MOCK_METHOD2(remove_namespace, void(int64_t, const std::string &)); + MOCK_METHOD3(add_or_update_attribute, void(int64_t, const std::string&, const service_daemon::AttributeValue&)); @@ -436,6 +439,20 @@ public: EXPECT_CALL(mock_namespace_replayer, handle_instances_removed(_)); } + void expect_service_daemon_add_namespace( + MockServiceDaemon &mock_service_daemon, + const std::string& namespace_name) { + EXPECT_CALL(mock_service_daemon, + add_namespace(m_local_io_ctx.get_id(), namespace_name)); + } + + void expect_service_daemon_remove_namespace( + MockServiceDaemon &mock_service_daemon, + const std::string& namespace_name) { + EXPECT_CALL(mock_service_daemon, + remove_namespace(m_local_io_ctx.get_id(), namespace_name)); + } + void expect_service_daemon_add_or_update_attribute( MockServiceDaemon &mock_service_daemon, const std::string& key, const service_daemon::AttributeValue& value) { @@ -446,6 +463,12 @@ public: MockServiceDaemon &mock_service_daemon, const std::string& key) { EXPECT_CALL(mock_service_daemon, remove_attribute(_, key)); } + + void expect_service_daemon_add_or_update_instance_id_attribute( + MockServiceDaemon &mock_service_daemon, const std::string &instance_id) { + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, "instance_id", {instance_id}); + } }; TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { @@ -485,6 +508,10 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -537,6 +564,10 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -615,12 +646,17 @@ TEST_F(TestMockPoolReplayer, Namespaces) { expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); C_SaferCond on_ns1_init; expect_namespace_replayer_init(*mock_ns1_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns1"); expect_namespace_replayer_handle_update_leader(*mock_ns1_namespace_replayer, "", &on_ns1_init); @@ -639,6 +675,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { ASSERT_EQ(0, on_acquire.wait()); expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); C_SaferCond on_ns2_acquire; expect_namespace_replayer_handle_acquire_leader( *mock_ns2_namespace_replayer, 0, &on_ns2_acquire); @@ -649,6 +686,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { ASSERT_EQ(0, on_ns2_acquire.wait()); C_SaferCond on_ns2_shut_down; + expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, &on_ns2_shut_down); mock_namespace.remove("ns2"); @@ -665,6 +703,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { mock_leader_watcher->listener->pre_release_handler(&on_release); ASSERT_EQ(0, on_release.wait()); + expect_service_daemon_remove_namespace(mock_service_daemon, "ns1"); expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer); expect_leader_watcher_shut_down(*mock_leader_watcher); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); @@ -719,6 +758,10 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -750,11 +793,12 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { C_SaferCond on_ns2_init; expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_handle_update_leader(*mock_ns2_namespace_replayer, "", &on_ns2_init); mock_namespace.add("ns2"); ASSERT_EQ(0, on_ns2_init.wait()); - + expect_service_daemon_add_or_update_attribute( mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); expect_namespace_replayer_handle_acquire_leader( @@ -766,6 +810,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { [&mock_namespace](int) { mock_namespace.remove("ns2"); }); + expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, ctx); mock_namespace.add("ns2"); @@ -782,8 +827,10 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { on_ns3_shut_down.complete(0); }); expect_namespace_replayer_init(*mock_ns3_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns3"); expect_namespace_replayer_handle_acquire_leader(*mock_ns3_namespace_replayer, -EINVAL); + expect_service_daemon_remove_namespace(mock_service_daemon, "ns3"); expect_namespace_replayer_shut_down(*mock_ns3_namespace_replayer, ctx); mock_namespace.add("ns3"); ASSERT_EQ(0, on_ns3_shut_down.wait()); diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 9795f8073f1..5e6fc66366d 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -382,15 +382,15 @@ void InstanceReplayer::start_image_replayers(int r) { start_image_replayer(current_it->second); } - // TODO: add namespace support to service daemon - if (m_local_io_ctx.get_namespace().empty()) { - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_ASSIGNED_COUNT_KEY, image_count); - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_WARNING_COUNT_KEY, warning_count); - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_ERROR_COUNT_KEY, error_count); - } + m_service_daemon->add_or_update_namespace_attribute( + m_local_io_ctx.get_id(), m_local_io_ctx.get_namespace(), + SERVICE_DAEMON_ASSIGNED_COUNT_KEY, image_count); + m_service_daemon->add_or_update_namespace_attribute( + m_local_io_ctx.get_id(), m_local_io_ctx.get_namespace(), + SERVICE_DAEMON_WARNING_COUNT_KEY, warning_count); + m_service_daemon->add_or_update_namespace_attribute( + m_local_io_ctx.get_id(), m_local_io_ctx.get_namespace(), + SERVICE_DAEMON_ERROR_COUNT_KEY, error_count); m_async_op_tracker.finish_op(); } diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index b22a547dc76..983682159dc 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -29,7 +29,6 @@ using ::operator<<; namespace { -const std::string SERVICE_DAEMON_INSTANCE_ID_KEY("instance_id"); const std::string SERVICE_DAEMON_LOCAL_COUNT_KEY("image_local_count"); const std::string SERVICE_DAEMON_REMOTE_COUNT_KEY("image_remote_count"); @@ -183,16 +182,14 @@ void NamespaceReplayer::handle_update(const std::string &mirror_uuid, << "added_count=" << added_image_ids.size() << ", " << "removed_count=" << removed_image_ids.size() << dendl; - // TODO: add namespace support to service daemon - if (m_local_io_ctx.get_namespace().empty()) { - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_LOCAL_COUNT_KEY, - m_local_pool_watcher->get_image_count()); - if (m_remote_pool_watcher) { - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_REMOTE_COUNT_KEY, - m_remote_pool_watcher->get_image_count()); - } + m_service_daemon->add_or_update_namespace_attribute( + m_local_io_ctx.get_id(), m_local_io_ctx.get_namespace(), + SERVICE_DAEMON_LOCAL_COUNT_KEY, m_local_pool_watcher->get_image_count()); + if (m_remote_pool_watcher) { + m_service_daemon->add_or_update_namespace_attribute( + m_local_io_ctx.get_id(), m_local_io_ctx.get_namespace(), + SERVICE_DAEMON_REMOTE_COUNT_KEY, + m_remote_pool_watcher->get_image_count()); } std::set added_global_image_ids; @@ -397,13 +394,6 @@ void NamespaceReplayer::handle_init_instance_watcher(int r) { return; } - // TODO: add namespace support to service daemon - if (m_local_io_ctx.get_namespace().empty()) { - m_service_daemon->add_or_update_attribute( - m_local_io_ctx.get_id(), SERVICE_DAEMON_INSTANCE_ID_KEY, - m_instance_watcher->get_instance_id()); - } - ceph_assert(m_on_finish != nullptr); m_threads->work_queue->queue(m_on_finish); m_on_finish = nullptr; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 74bb06c48ce..75ddc5fd956 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -31,6 +31,7 @@ using ::operator<<; namespace { +const std::string SERVICE_DAEMON_INSTANCE_ID_KEY("instance_id"); const std::string SERVICE_DAEMON_LEADER_KEY("leader"); const std::vector UNIQUE_PEER_CONFIG_KEYS { @@ -352,6 +353,10 @@ void PoolReplayer::init(const std::string& site_name) { m_callout_id = service_daemon::CALLOUT_ID_NONE; } + m_service_daemon->add_or_update_attribute( + m_local_io_ctx.get_id(), SERVICE_DAEMON_INSTANCE_ID_KEY, + stringify(m_local_io_ctx.get_instance_id())); + m_pool_replayer_thread.create("pool replayer"); } @@ -586,6 +591,7 @@ void PoolReplayer::update_namespace_replayers() { delete namespace_replayer; ctx->complete(r); }); + m_service_daemon->remove_namespace(m_local_pool_id, it->first); namespace_replayer->shut_down(on_shut_down); it = m_namespace_replayers.erase(it); } else { @@ -611,6 +617,7 @@ void PoolReplayer::update_namespace_replayers() { } else { std::lock_guard locker{m_lock}; m_namespace_replayers[name] = namespace_replayer; + m_service_daemon->add_namespace(m_local_pool_id, name); } ctx->complete(r); }); @@ -716,6 +723,7 @@ void PoolReplayer::namespace_replayer_acquire_leader(const std::string &name, delete namespace_replayer; on_finish->complete(r); }); + m_service_daemon->remove_namespace(m_local_pool_id, name); namespace_replayer->shut_down(on_shut_down); return; } diff --git a/src/tools/rbd_mirror/ServiceDaemon.cc b/src/tools/rbd_mirror/ServiceDaemon.cc index 19f69ea88de..f3cabcc873a 100644 --- a/src/tools/rbd_mirror/ServiceDaemon.cc +++ b/src/tools/rbd_mirror/ServiceDaemon.cc @@ -108,6 +108,38 @@ void ServiceDaemon::remove_pool(int64_t pool_id) { schedule_update_status(); } +template +void ServiceDaemon::add_namespace(int64_t pool_id, + const std::string& namespace_name) { + dout(20) << "pool_id=" << pool_id << ", namespace=" << namespace_name + << dendl; + + std::lock_guard locker{m_lock}; + auto pool_it = m_pools.find(pool_id); + if (pool_it == m_pools.end()) { + return; + } + pool_it->second.ns_attributes[namespace_name]; + + // don't schedule update status as the namespace attributes are empty yet +} + +template +void ServiceDaemon::remove_namespace(int64_t pool_id, + const std::string& namespace_name) { + dout(20) << "pool_id=" << pool_id << ", namespace=" << namespace_name + << dendl; + { + std::lock_guard locker{m_lock}; + auto pool_it = m_pools.find(pool_id); + if (pool_it == m_pools.end()) { + return; + } + pool_it->second.ns_attributes.erase(namespace_name); + } + schedule_update_status(); +} + template uint64_t ServiceDaemon::add_or_update_callout(int64_t pool_id, uint64_t callout_id, @@ -172,6 +204,38 @@ void ServiceDaemon::add_or_update_attribute(int64_t pool_id, schedule_update_status(); } +template +void ServiceDaemon::add_or_update_namespace_attribute( + int64_t pool_id, const std::string& namespace_name, const std::string& key, + const AttributeValue& value) { + if (namespace_name.empty()) { + add_or_update_attribute(pool_id, key, value); + return; + } + + dout(20) << "pool_id=" << pool_id << ", " + << "namespace=" << namespace_name << ", " + << "key=" << key << ", " + << "value=" << value << dendl; + + { + std::lock_guard locker{m_lock}; + auto pool_it = m_pools.find(pool_id); + if (pool_it == m_pools.end()) { + return; + } + + auto ns_it = pool_it->second.ns_attributes.find(namespace_name); + if (ns_it == pool_it->second.ns_attributes.end()) { + return; + } + + ns_it->second[key] = value; + } + + schedule_update_status(); +} + template void ServiceDaemon::remove_attribute(int64_t pool_id, const std::string& key) { @@ -229,6 +293,19 @@ void ServiceDaemon::update_status() { AttributeDumpVisitor attribute_dump_visitor(&f, attribute.first); boost::apply_visitor(attribute_dump_visitor, attribute.second); } + + if (!pool_pair.second.ns_attributes.empty()) { + f.open_object_section("namespaces"); + for (auto& [ns, attributes] : pool_pair.second.ns_attributes) { + f.open_object_section(ns.c_str()); + for (auto& [key, value] : attributes) { + AttributeDumpVisitor attribute_dump_visitor(&f, key); + boost::apply_visitor(attribute_dump_visitor, value); + } + f.close_section(); // namespace + } + f.close_section(); // namespaces + } f.close_section(); // pool } f.close_section(); // pools diff --git a/src/tools/rbd_mirror/ServiceDaemon.h b/src/tools/rbd_mirror/ServiceDaemon.h index 68599df8099..8df1bda8f5c 100644 --- a/src/tools/rbd_mirror/ServiceDaemon.h +++ b/src/tools/rbd_mirror/ServiceDaemon.h @@ -30,6 +30,9 @@ public: void add_pool(int64_t pool_id, const std::string& pool_name); void remove_pool(int64_t pool_id); + void add_namespace(int64_t pool_id, const std::string& namespace_name); + void remove_namespace(int64_t pool_id, const std::string& namespace_name); + uint64_t add_or_update_callout(int64_t pool_id, uint64_t callout_id, service_daemon::CalloutLevel callout_level, const std::string& text); @@ -37,6 +40,9 @@ public: void add_or_update_attribute(int64_t pool_id, const std::string& key, const service_daemon::AttributeValue& value); + void add_or_update_namespace_attribute( + int64_t pool_id, const std::string& namespace_name, + const std::string& key, const service_daemon::AttributeValue& value); void remove_attribute(int64_t pool_id, const std::string& key); private: @@ -52,11 +58,13 @@ private: }; typedef std::map Callouts; typedef std::map Attributes; + typedef std::map NamespaceAttributes; struct Pool { std::string name; Callouts callouts; Attributes attributes; + NamespaceAttributes ns_attributes; Pool(const std::string& name) : name(name) { } -- 2.39.5