From: Jason Dillaman Date: Wed, 25 Sep 2019 16:40:04 +0000 (-0400) Subject: rbd-mirror: pass local site name through to namespace replayer X-Git-Tag: v15.1.0~1245^2~7 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2d229cf007448368ba1be35d0417f98d9d4343ad;p=ceph.git rbd-mirror: pass local site name through to namespace replayer Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index 9684bc98c8a2..bd52f9218145 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -414,7 +414,8 @@ TEST_F(TestMockNamespaceReplayer, Init_MirrorStatusUpdaterError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); + "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, nullptr, + nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -434,7 +435,8 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); + "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, nullptr, + nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -460,7 +462,8 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); + "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, nullptr, + nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -487,7 +490,7 @@ TEST_F(TestMockNamespaceReplayer, Init) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, + "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, &mock_service_daemon, nullptr); C_SaferCond on_init; @@ -526,7 +529,7 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, + "remote mirror uuid", "siteA", m_mock_threads, nullptr, nullptr, &mock_service_daemon, nullptr); C_SaferCond on_init; diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 3b952eda738a..039ab7292e4c 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -132,6 +132,7 @@ struct NamespaceReplayer { librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, + const std::string &site_name, Threads *threads, Throttler *image_sync_throttler, Throttler *image_deletion_throttler, @@ -486,7 +487,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { MockServiceDaemon mock_service_daemon; MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); - pool_replayer.init(); + pool_replayer.init("siteA"); ASSERT_TRUE(remote_cct != nullptr); ASSERT_EQ("123", remote_cct->_conf.get_val("mon_host")); @@ -538,7 +539,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { MockServiceDaemon mock_service_daemon; MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); - pool_replayer.init(); + pool_replayer.init("siteA"); expect_service_daemon_add_or_update_attribute( mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); @@ -616,7 +617,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { MockServiceDaemon mock_service_daemon; MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); - pool_replayer.init(); + pool_replayer.init("siteA"); C_SaferCond on_ns1_init; expect_namespace_replayer_init(*mock_ns1_namespace_replayer, 0); @@ -720,7 +721,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { MockServiceDaemon mock_service_daemon; MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, m_local_io_ctx.get_id(), peer_spec, {}); - pool_replayer.init(); + pool_replayer.init("siteA"); // test namespace replayer init fails for non leader diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc index 9b2596bbf732..1a6604484f47 100644 --- a/src/tools/rbd_mirror/Mirror.cc +++ b/src/tools/rbd_mirror/Mirror.cc @@ -548,7 +548,8 @@ void Mirror::run() std::unique_lock l{m_lock}; if (!m_manual_stop) { if (refresh_pools) { - update_pool_replayers(m_local_cluster_watcher->get_pool_peers()); + update_pool_replayers(m_local_cluster_watcher->get_pool_peers(), + m_local_cluster_watcher->get_site_name()); } m_cache_manager_handler->run_cache_manager(); } @@ -658,7 +659,8 @@ void Mirror::release_leader() } } -void Mirror::update_pool_replayers(const PoolPeers &pool_peers) +void Mirror::update_pool_replayers(const PoolPeers &pool_peers, + const std::string& site_name) { dout(20) << "enter" << dendl; ceph_assert(ceph_mutex_is_locked(m_lock)); @@ -685,16 +687,23 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers) auto pool_replayers_it = m_pool_replayers.find(pool_peer); if (pool_replayers_it != m_pool_replayers.end()) { auto& pool_replayer = pool_replayers_it->second; - if (pool_replayer->is_blacklisted()) { + if (!m_site_name.empty() && !site_name.empty() && + m_site_name != site_name) { + dout(0) << "restarting pool replayer for " << peer << " due to " + << "updated site name" << dendl; + // TODO: make async + pool_replayer->shut_down(); + pool_replayer->init(site_name); + } else if (pool_replayer->is_blacklisted()) { derr << "restarting blacklisted pool replayer for " << peer << dendl; // TODO: make async pool_replayer->shut_down(); - pool_replayer->init(); + pool_replayer->init(site_name); } else if (!pool_replayer->is_running()) { derr << "restarting failed pool replayer for " << peer << dendl; // TODO: make async pool_replayer->shut_down(); - pool_replayer->init(); + pool_replayer->init(site_name); } } else { dout(20) << "starting pool replayer for " << peer << dendl; @@ -704,13 +713,15 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers) m_args)); // TODO: make async - pool_replayer->init(); + pool_replayer->init(site_name); m_pool_replayers.emplace(pool_peer, std::move(pool_replayer)); } } // TODO currently only support a single peer } + + m_site_name = site_name; } } // namespace mirror diff --git a/src/tools/rbd_mirror/Mirror.h b/src/tools/rbd_mirror/Mirror.h index 9b1496285adb..28f71c54dc2d 100644 --- a/src/tools/rbd_mirror/Mirror.h +++ b/src/tools/rbd_mirror/Mirror.h @@ -57,7 +57,8 @@ private: typedef ClusterWatcher::PoolPeers PoolPeers; typedef std::pair PoolPeer; - void update_pool_replayers(const PoolPeers &pool_peers); + void update_pool_replayers(const PoolPeers &pool_peers, + const std::string& site_name); void create_cache_manager(); void run_cache_manager(utime_t *next_run_interval); @@ -77,6 +78,7 @@ private: std::atomic m_stopping = { false }; bool m_manual_stop = false; MirrorAdminSocketHook *m_asok_hook; + std::string m_site_name; }; } // namespace mirror diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 7ed94cac01a1..8919945cad13 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -40,11 +40,13 @@ NamespaceReplayer::NamespaceReplayer( const std::string &name, librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, - Threads *threads, Throttler *image_sync_throttler, - Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, + const std::string &local_site_name, Threads *threads, + Throttler *image_sync_throttler, Throttler *image_deletion_throttler, + ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler) : m_local_mirror_uuid(local_mirror_uuid), m_remote_mirror_uuid(remote_mirror_uuid), + m_local_site_name(local_site_name), m_threads(threads), m_image_sync_throttler(image_sync_throttler), m_image_deletion_throttler(image_deletion_throttler), m_service_daemon(service_daemon), diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h index 3455f59fd8fb..20a5659c82a9 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.h +++ b/src/tools/rbd_mirror/NamespaceReplayer.h @@ -48,15 +48,17 @@ public: librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, + const std::string &local_site_name, Threads *threads, Throttler *image_sync_throttler, Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler) { return new NamespaceReplayer(name, local_ioctx, remote_ioctx, - local_mirror_uuid, remote_mirror_uuid, threads, - image_sync_throttler, image_deletion_throttler, - service_daemon, cache_manager_handler); + local_mirror_uuid, remote_mirror_uuid, + local_site_name, threads, image_sync_throttler, + image_deletion_throttler, service_daemon, + cache_manager_handler); } NamespaceReplayer(const std::string &name, @@ -64,6 +66,7 @@ public: librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, + const std::string &local_site_name, Threads *threads, Throttler *image_sync_throttler, Throttler *image_deletion_throttler, @@ -252,6 +255,7 @@ private: librados::IoCtx m_remote_io_ctx; std::string m_local_mirror_uuid; std::string m_remote_mirror_uuid; + std::string m_local_site_name; Threads *m_threads; Throttler *m_image_sync_throttler; Throttler *m_image_deletion_throttler; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index e518f2d4f95a..e4c6e857f9d3 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -246,12 +246,13 @@ bool PoolReplayer::is_running() const { } template -void PoolReplayer::init() { +void PoolReplayer::init(const std::string& site_name) { ceph_assert(!m_pool_replayer_thread.is_started()); // reset state m_stopping = false; m_blacklisted = false; + m_site_name = site_name; dout(10) << "replaying for " << m_peer << dendl; int r = init_rados(g_ceph_context->_conf->cluster, @@ -317,8 +318,9 @@ void PoolReplayer::init() { m_default_namespace_replayer.reset(NamespaceReplayer::create( "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, - m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), - m_service_daemon, m_cache_manager_handler)); + m_site_name, m_threads, m_image_sync_throttler.get(), + m_image_deletion_throttler.get(), m_service_daemon, + m_cache_manager_handler)); C_SaferCond on_init; m_default_namespace_replayer->init(&on_init); @@ -595,7 +597,7 @@ void PoolReplayer::update_namespace_replayers() { for (auto &name : mirroring_namespaces) { auto namespace_replayer = NamespaceReplayer::create( name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, - m_threads, m_image_sync_throttler.get(), + m_site_name, m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), m_service_daemon, m_cache_manager_handler); auto on_init = new LambdaContext( diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 9534d755c5f8..312de1f44399 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -53,7 +53,7 @@ public: bool is_leader() const; bool is_running() const; - void init(); + void init(const std::string& site_name); void shut_down(); void run(); @@ -190,6 +190,7 @@ private: mutable ceph::mutex m_lock; ceph::condition_variable m_cond; + std::string m_site_name; bool m_stopping = false; bool m_manual_stop = false; bool m_blacklisted = false;