From: Jason Dillaman Date: Tue, 11 Jul 2017 16:44:02 +0000 (-0400) Subject: rbd-mirror: skeleton integration with service daemon API X-Git-Tag: v12.1.2~151^2~18 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3013c8ad354482625597638e1b2501b7f37c6631;p=ceph.git rbd-mirror: skeleton integration with service daemon API Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_ClusterWatcher.cc b/src/test/rbd_mirror/test_ClusterWatcher.cc index 2fd894c702e7..61f67a5f2279 100644 --- a/src/test/rbd_mirror/test_ClusterWatcher.cc +++ b/src/test/rbd_mirror/test_ClusterWatcher.cc @@ -7,6 +7,7 @@ #include "librbd/internal.h" #include "librbd/api/Mirror.h" #include "tools/rbd_mirror/ClusterWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/types.h" #include "test/rbd_mirror/test_fixture.h" #include "test/librados/test.h" @@ -34,7 +35,10 @@ public: { m_cluster = std::make_shared(); EXPECT_EQ("", connect_cluster_pp(*m_cluster)); - m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock)); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + m_cluster)); + m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock, + m_service_daemon.get())); } ~TestClusterWatcher() override { @@ -126,8 +130,9 @@ public: ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers()); } - Mutex m_lock; RadosRef m_cluster; + Mutex m_lock; + unique_ptr> m_service_daemon; unique_ptr m_cluster_watcher; set m_pools; diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc index 24f525749494..0f0b44fdb149 100644 --- a/src/test/rbd_mirror/test_ImageDeleter.cc +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -18,6 +18,7 @@ #include "cls/rbd/cls_rbd_client.h" #include "tools/rbd_mirror/ImageDeleter.h" #include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/Threads.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" @@ -61,12 +62,15 @@ public: void SetUp() override { TestFixture::SetUp(); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + _rados)); librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); m_deleter = new rbd::mirror::ImageDeleter<>(m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock); + &m_threads->timer_lock, + m_service_daemon.get()); EXPECT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, 1 << 20)); ImageCtx *ictx = new ImageCtx(m_image_name, "", "", m_local_io_ctx, @@ -210,6 +214,7 @@ public: librbd::RBD rbd; std::string m_local_image_id; + std::unique_ptr> m_service_daemon; rbd::mirror::ImageDeleter<> *m_deleter; }; diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 9b9791c48b46..810ebff41a7e 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -35,10 +35,11 @@ #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ReadResult.h" #include "tools/rbd_mirror/types.h" +#include "tools/rbd_mirror/ImageDeleter.h" #include "tools/rbd_mirror/ImageReplayer.h" #include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/Threads.h" -#include "tools/rbd_mirror/ImageDeleter.h" #include "test/librados/test.h" #include "gtest/gtest.h" @@ -115,8 +116,11 @@ public: m_threads = new rbd::mirror::Threads<>(reinterpret_cast( m_local_ioctx.cct())); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + m_local_cluster)); m_image_deleter.reset(new rbd::mirror::ImageDeleter<>( - m_threads->work_queue, m_threads->timer, &m_threads->timer_lock)); + m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, + m_service_daemon.get())); m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( m_local_ioctx, m_threads->work_queue, nullptr); m_instance_watcher->handle_acquire_leader(); @@ -366,6 +370,7 @@ public: static int _image_number; rbd::mirror::Threads<> *m_threads = nullptr; + unique_ptr> m_service_daemon; std::unique_ptr> m_image_deleter; std::shared_ptr m_local_cluster; librados::Rados m_remote_cluster; diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index f577aa47ba42..9e786fc75704 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -7,6 +7,7 @@ #include "tools/rbd_mirror/ImageReplayer.h" #include "tools/rbd_mirror/InstanceWatcher.h" #include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/Threads.h" namespace librbd { @@ -46,6 +47,10 @@ struct ImageDeleter { void(int64_t, const std::string&, Context*, bool)); }; +template<> +struct ServiceDaemon { +}; + template<> struct InstanceWatcher { }; @@ -120,6 +125,7 @@ public: typedef ImageReplayer MockImageReplayer; typedef InstanceReplayer MockInstanceReplayer; typedef InstanceWatcher MockInstanceWatcher; + typedef ServiceDaemon MockServiceDaemon; typedef Threads MockThreads; void SetUp() override { @@ -147,11 +153,12 @@ public: }; TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { + MockServiceDaemon mock_service_daemon; MockImageDeleter mock_image_deleter; MockInstanceWatcher mock_instance_watcher; MockImageReplayer mock_image_replayer; MockInstanceReplayer instance_replayer( - m_mock_threads, &mock_image_deleter, + m_mock_threads, &mock_service_daemon, &mock_image_deleter, rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), "local_mirror_uuid", m_local_io_ctx.get_id()); diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 1a9d0fb1aab4..914dfc7b8091 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -16,6 +16,7 @@ set(rbd_mirror_internal MirrorStatusWatcher.cc PoolReplayer.cc PoolWatcher.cc + ServiceDaemon.cc Threads.cc types.cc image_replayer/BootstrapRequest.cc diff --git a/src/tools/rbd_mirror/ClusterWatcher.cc b/src/tools/rbd_mirror/ClusterWatcher.cc index 9bb96b166dd2..f17a5ea69280 100644 --- a/src/tools/rbd_mirror/ClusterWatcher.cc +++ b/src/tools/rbd_mirror/ClusterWatcher.cc @@ -27,9 +27,9 @@ using librados::IoCtx; namespace rbd { namespace mirror { -ClusterWatcher::ClusterWatcher(RadosRef cluster, Mutex &lock) : - m_lock(lock), - m_cluster(cluster) +ClusterWatcher::ClusterWatcher(RadosRef cluster, Mutex &lock, + ServiceDaemon* service_daemon) + : m_cluster(cluster), m_lock(lock), m_service_daemon(service_daemon) { } diff --git a/src/tools/rbd_mirror/ClusterWatcher.h b/src/tools/rbd_mirror/ClusterWatcher.h index f101de07ba47..e49202f2c2a1 100644 --- a/src/tools/rbd_mirror/ClusterWatcher.h +++ b/src/tools/rbd_mirror/ClusterWatcher.h @@ -14,9 +14,13 @@ #include "include/rados/librados.hpp" #include "types.h" +namespace librbd { struct ImageCtx; } + namespace rbd { namespace mirror { +template class ServiceDaemon; + /** * Tracks mirroring configuration for pools in a single * cluster. @@ -27,7 +31,8 @@ public: typedef std::map PoolPeers; typedef std::set PoolNames; - ClusterWatcher(RadosRef cluster, Mutex &lock); + ClusterWatcher(RadosRef cluster, Mutex &lock, + ServiceDaemon* service_daemon); ~ClusterWatcher() = default; ClusterWatcher(const ClusterWatcher&) = delete; ClusterWatcher& operator=(const ClusterWatcher&) = delete; @@ -37,8 +42,10 @@ public: const PoolPeers& get_pool_peers() const; private: - Mutex &m_lock; RadosRef m_cluster; + Mutex &m_lock; + ServiceDaemon* m_service_daemon; + PoolPeers m_pool_peers; void read_pool_peers(PoolPeers *pool_peers, PoolNames *pool_names); diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index a0047d353c4d..611b0c20948b 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -137,9 +137,10 @@ private: template ImageDeleter::ImageDeleter(ContextWQ *work_queue, SafeTimer *timer, - Mutex *timer_lock) - : m_running(true), - m_work_queue(work_queue), + Mutex *timer_lock, + ServiceDaemon* service_daemon) + : m_work_queue(work_queue), + m_service_daemon(service_daemon), m_delete_lock("rbd::mirror::ImageDeleter::Delete"), m_image_deleter_thread(this), m_failed_timer(timer), diff --git a/src/tools/rbd_mirror/ImageDeleter.h b/src/tools/rbd_mirror/ImageDeleter.h index e55337680f09..039c7a11c03a 100644 --- a/src/tools/rbd_mirror/ImageDeleter.h +++ b/src/tools/rbd_mirror/ImageDeleter.h @@ -32,6 +32,8 @@ namespace librbd { struct ImageCtx; } namespace rbd { namespace mirror { +template class ServiceDaemon; + /** * Manage deletion of non-primary images. */ @@ -40,7 +42,8 @@ class ImageDeleter { public: static const int EISPRM = 1000; - ImageDeleter(ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock); + ImageDeleter(ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, + ServiceDaemon* service_daemon); ~ImageDeleter(); ImageDeleter(const ImageDeleter&) = delete; ImageDeleter& operator=(const ImageDeleter&) = delete; @@ -100,9 +103,10 @@ private: bool print_failure_info=false); }; - std::atomic m_running { 0 }; + std::atomic m_running { 1 }; ContextWQ *m_work_queue; + ServiceDaemon* m_service_daemon; std::deque > m_delete_queue; Mutex m_delete_lock; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 74efb295759e..5dbddc16f6c6 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -24,12 +24,12 @@ using librbd::util::create_context_callback; template InstanceReplayer::InstanceReplayer( - Threads *threads, ImageDeleter* image_deleter, - RadosRef local_rados, const std::string &local_mirror_uuid, - int64_t local_pool_id) - : m_threads(threads), m_image_deleter(image_deleter), - m_local_rados(local_rados), m_local_mirror_uuid(local_mirror_uuid), - m_local_pool_id(local_pool_id), + Threads *threads, ServiceDaemon* service_daemon, + ImageDeleter* image_deleter, RadosRef local_rados, + const std::string &local_mirror_uuid, int64_t local_pool_id) + : m_threads(threads), m_service_daemon(service_daemon), + m_image_deleter(image_deleter), m_local_rados(local_rados), + m_local_mirror_uuid(local_mirror_uuid), m_local_pool_id(local_pool_id), m_lock("rbd::mirror::InstanceReplayer " + stringify(local_pool_id)) { } diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index b704f1785cf1..ecd7d11f4a7c 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -20,6 +20,7 @@ namespace mirror { template class ImageDeleter; template class ImageReplayer; template class InstanceWatcher; +template class ServiceDaemon; template struct Threads; template @@ -27,17 +28,19 @@ class InstanceReplayer { public: static InstanceReplayer* create( Threads *threads, + ServiceDaemon* service_daemon, ImageDeleter* image_deleter, RadosRef local_rados, const std::string &local_mirror_uuid, int64_t local_pool_id) { - return new InstanceReplayer(threads, image_deleter, local_rados, - local_mirror_uuid, local_pool_id); + return new InstanceReplayer(threads, service_daemon, image_deleter, + local_rados, local_mirror_uuid, local_pool_id); } void destroy() { delete this; } InstanceReplayer(Threads *threads, + ServiceDaemon* service_daemon, ImageDeleter* image_deleter, RadosRef local_rados, const std::string &local_mirror_uuid, int64_t local_pool_id); @@ -109,6 +112,7 @@ private: typedef std::set Peers; Threads *m_threads; + ServiceDaemon* m_service_daemon; ImageDeleter* m_image_deleter; RadosRef m_local_rados; std::string m_local_mirror_uuid; diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc index 4697d37b6735..20036be055c1 100644 --- a/src/tools/rbd_mirror/Mirror.cc +++ b/src/tools/rbd_mirror/Mirror.cc @@ -202,6 +202,7 @@ Mirror::Mirror(CephContext *cct, const std::vector &args) : m_args(args), m_lock("rbd::mirror::Mirror"), m_local(new librados::Rados()), + m_service_daemon(m_cct, m_local), m_asok_hook(new MirrorAdminSocketHook(cct, this)) { cct->lookup_or_create_singleton_object >( @@ -236,12 +237,19 @@ int Mirror::init() return r; } - m_local_cluster_watcher.reset(new ClusterWatcher(m_local, m_lock)); + r = m_service_daemon.init(); + if (r < 0) { + derr << "error registering service daemon: " << cpp_strerror(r) << dendl; + return r; + } + + m_local_cluster_watcher.reset(new ClusterWatcher(m_local, m_lock, + &m_service_daemon)); m_image_deleter.reset(new ImageDeleter<>(m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock)); - + &m_threads->timer_lock, + &m_service_daemon)); return r; } @@ -399,7 +407,8 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers) if (m_pool_replayers.find(pool_peer) == m_pool_replayers.end()) { dout(20) << "starting pool replayer for " << peer << dendl; unique_ptr pool_replayer(new PoolReplayer( - m_threads, m_image_deleter.get(), kv.first, peer, m_args)); + m_threads, &m_service_daemon, m_image_deleter.get(), kv.first, + peer, m_args)); // TODO: make async, and retry connecting within pool replayer int r = pool_replayer->init(); diff --git a/src/tools/rbd_mirror/Mirror.h b/src/tools/rbd_mirror/Mirror.h index 17d712f37d9f..8558f82c821c 100644 --- a/src/tools/rbd_mirror/Mirror.h +++ b/src/tools/rbd_mirror/Mirror.h @@ -11,6 +11,7 @@ #include "PoolReplayer.h" #include "ImageDeleter.h" #include "types.h" +#include "ServiceDaemon.h" #include #include @@ -61,6 +62,7 @@ private: Mutex m_lock; Cond m_cond; RadosRef m_local; + ServiceDaemon m_service_daemon; // monitor local cluster for config changes in peers std::unique_ptr m_local_cluster_watcher; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index f63828eadce7..ede06fae76bb 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -206,10 +206,12 @@ private: } // anonymous namespace PoolReplayer::PoolReplayer(Threads *threads, + ServiceDaemon* service_daemon, ImageDeleter<>* image_deleter, int64_t local_pool_id, const peer_t &peer, const std::vector &args) : m_threads(threads), + m_service_daemon(service_daemon), m_image_deleter(image_deleter), m_lock(stringify("rbd::mirror::PoolReplayer ") + stringify(peer)), m_peer(peer), @@ -303,9 +305,9 @@ int PoolReplayer::init() dout(20) << "connected to " << m_peer << dendl; - m_instance_replayer.reset( - InstanceReplayer<>::create(m_threads, m_image_deleter, m_local_rados, - local_mirror_uuid, m_local_pool_id)); + m_instance_replayer.reset(InstanceReplayer<>::create( + m_threads, m_service_daemon, m_image_deleter, m_local_rados, + local_mirror_uuid, m_local_pool_id)); m_instance_replayer->init(); m_instance_replayer->add_peer(m_peer.uuid, m_remote_io_ctx); diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 1c4170a0a0b7..d23b0d66a744 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -29,9 +29,10 @@ namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { -template struct Threads; template class InstanceReplayer; template class InstanceWatcher; +template class ServiceDaemon; +template struct Threads; /** * Controls mirroring for a single remote cluster. @@ -39,6 +40,7 @@ template class InstanceWatcher; class PoolReplayer { public: PoolReplayer(Threads *threads, + ServiceDaemon* service_daemon, ImageDeleter<>* image_deleter, int64_t local_pool_id, const peer_t &peer, const std::vector &args); @@ -102,6 +104,7 @@ private: void handle_update_leader(const std::string &leader_instance_id); Threads *m_threads; + ServiceDaemon* m_service_daemon; ImageDeleter<>* m_image_deleter; mutable Mutex m_lock; Cond m_cond; diff --git a/src/tools/rbd_mirror/ServiceDaemon.cc b/src/tools/rbd_mirror/ServiceDaemon.cc new file mode 100644 index 000000000000..2d0ecae07c64 --- /dev/null +++ b/src/tools/rbd_mirror/ServiceDaemon.cc @@ -0,0 +1,47 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "include/stringify.h" +#include "common/ceph_context.h" +#include "common/config.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::ServiceDaemon: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { + +namespace { + +const std::string RBD_MIRROR_AUTH_ID_PREFIX("rbd-mirror."); + +} // anonymous namespace + +template +int ServiceDaemon::init() { + std::string name = m_cct->_conf->name.get_id(); + if (name.find(RBD_MIRROR_AUTH_ID_PREFIX) == 0) { + name = name.substr(RBD_MIRROR_AUTH_ID_PREFIX.size()); + } + + std::map service_metadata = { + {"instance_id", stringify(m_rados->get_instance_id())} + }; + int r = m_rados->service_daemon_register("rbd-mirror", name, + service_metadata); + if (r < 0) { + return r; + } + + return 0; +} + +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::ServiceDaemon; diff --git a/src/tools/rbd_mirror/ServiceDaemon.h b/src/tools/rbd_mirror/ServiceDaemon.h new file mode 100644 index 000000000000..73c3b9b17fa8 --- /dev/null +++ b/src/tools/rbd_mirror/ServiceDaemon.h @@ -0,0 +1,37 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_SERVICE_DAEMON_H +#define CEPH_RBD_MIRROR_SERVICE_DAEMON_H + +#include "tools/rbd_mirror/types.h" +#include + +struct CephContext; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +template +class ServiceDaemon { +public: + ServiceDaemon(CephContext *cct, RadosRef rados) + : m_cct(cct), m_rados(rados) { + } + + int init(); + +private: + CephContext *m_cct; + RadosRef m_rados; + +}; + +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::ServiceDaemon; + +#endif // CEPH_RBD_MIRROR_SERVICE_DAEMON_H