From: Jason Dillaman Date: Thu, 20 Jul 2017 20:39:51 +0000 (-0400) Subject: rbd-mirror: co-opt image acquire/release notifications X-Git-Tag: v13.0.0~218^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d49945242e050a266c4d163d37554eeba07110d4;p=ceph.git rbd-mirror: co-opt image acquire/release notifications This notifications will now be used by the policy when a (re)map is required. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 57ceeff4740..d8e159f5f3d 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -150,8 +150,7 @@ public: m_threads.get(), m_image_deleter.get(), m_instance_watcher, rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), m_local_mirror_uuid, m_local_ioctx.get_id(), m_global_image_id); - m_replayer->add_remote_image(m_remote_mirror_uuid, m_remote_image_id, - m_remote_ioctx); + m_replayer->add_peer("peer uuid", m_remote_ioctx); } void start() diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index b5d4ce6e23a..a46761d82d3 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -493,8 +493,7 @@ public: m_threads, &mock_image_deleter, &m_instance_watcher, rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id"); - m_image_replayer->add_remote_image( - "remote_mirror_uuid", m_remote_image_ctx->id, m_remote_io_ctx); + m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx); } librbd::ImageCtx *m_remote_image_ctx; diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index 6e2d58c0aa0..10f58232249 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -91,13 +91,7 @@ struct ImageReplayer { MOCK_METHOD0(restart, void()); MOCK_METHOD0(flush, void()); MOCK_METHOD2(print_status, void(Formatter *, stringstream *)); - MOCK_METHOD3(add_remote_image, void(const std::string &, - const std::string &, - librados::IoCtx &)); - MOCK_METHOD3(remove_remote_image, void(const std::string &, - const std::string &, - bool)); - MOCK_METHOD0(remote_images_empty, bool()); + MOCK_METHOD2(add_peer, void(const std::string &, librados::IoCtx &)); MOCK_METHOD0(get_global_image_id, const std::string &()); MOCK_METHOD0(get_local_image_id, const std::string &()); MOCK_METHOD0(is_running, bool()); @@ -180,14 +174,13 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { InSequence seq; instance_replayer.init(); - instance_replayer.add_peer("remote_mirror_uuid", m_remote_io_ctx); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); // Acquire C_SaferCond on_acquire; - EXPECT_CALL(mock_image_replayer, add_remote_image("remote_mirror_uuid", - "remote_image_id", _)); + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); EXPECT_CALL(mock_image_replayer, is_stopped()) .WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, start(nullptr, false)); @@ -201,11 +194,6 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { C_SaferCond on_release; - EXPECT_CALL(mock_image_replayer, - remove_remote_image("remote_mirror_uuid", "remote_image_id", - false)); - EXPECT_CALL(mock_image_replayer, remote_images_empty()) - .WillOnce(Return(true)); EXPECT_CALL(mock_image_replayer, is_stopped()) .WillOnce(Return(false)); EXPECT_CALL(mock_image_replayer, is_running()) diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 5842dd4fb4f..434f50459ca 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -343,32 +343,15 @@ image_replayer::HealthState ImageReplayer::get_health_state() const { } template -void ImageReplayer::add_remote_image(const std::string &mirror_uuid, - const std::string &image_id, - librados::IoCtx &io_ctx) { +void ImageReplayer::add_peer(const std::string &peer_uuid, + librados::IoCtx &io_ctx) { Mutex::Locker locker(m_lock); - - RemoteImage remote_image(mirror_uuid, image_id, io_ctx); - auto it = m_remote_images.find(remote_image); - if (it == m_remote_images.end()) { - m_remote_images.insert(remote_image); + auto it = m_peers.find({peer_uuid}); + if (it == m_peers.end()) { + m_peers.insert({peer_uuid, io_ctx}); } } -template -void ImageReplayer::remove_remote_image(const std::string &mirror_uuid, - const std::string &image_id, - bool schedule_delete) { - Mutex::Locker locker(m_lock); - m_remote_images.erase({mirror_uuid, image_id}); -} - -template -bool ImageReplayer::remote_images_empty() const { - Mutex::Locker locker(m_lock); - return m_remote_images.empty(); -} - template void ImageReplayer::set_state_description(int r, const std::string &desc) { dout(20) << r << " " << desc << dendl; @@ -459,13 +442,13 @@ void ImageReplayer::handle_prepare_local_image(int r) { template void ImageReplayer::prepare_remote_image() { dout(20) << dendl; - if (m_remote_images.empty()) { + if (m_peers.empty()) { on_start_fail(-EREMOTEIO, "waiting for primary remote image"); return; } // TODO bootstrap will need to support multiple remote images - m_remote_image = *m_remote_images.begin(); + m_remote_image = {*m_peers.begin()}; Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_prepare_remote_image>(this); diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index fe0be00b563..d15adb280b0 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -97,13 +97,7 @@ public: image_replayer::HealthState get_health_state() const; - void add_remote_image(const std::string &remote_mirror_uuid, - const std::string &remote_image_id, - librados::IoCtx &remote_io_ctx); - void remove_remote_image(const std::string &remote_mirror_uuid, - const std::string &remote_image_id, - bool schedule_delete); - bool remote_images_empty() const; + void add_peer(const std::string &peer_uuid, librados::IoCtx &remote_io_ctx); inline int64_t get_local_pool_id() const { return m_local_pool_id; @@ -227,30 +221,10 @@ private: RemoteImage() { } - RemoteImage(const std::string &mirror_uuid, - const std::string &image_id) - : mirror_uuid(mirror_uuid), image_id(image_id) { - } - RemoteImage(const std::string &mirror_uuid, - const std::string &image_id, - librados::IoCtx &io_ctx) - : mirror_uuid(mirror_uuid), image_id(image_id), io_ctx(io_ctx) { - } - - inline bool operator<(const RemoteImage &rhs) const { - if (mirror_uuid != rhs.mirror_uuid) { - return mirror_uuid < rhs.mirror_uuid; - } else { - return image_id < rhs.image_id; - } - } - inline bool operator==(const RemoteImage &rhs) const { - return (mirror_uuid == rhs.mirror_uuid && image_id == rhs.image_id); + RemoteImage(const Peer& peer) : io_ctx(peer.io_ctx) { } }; - typedef std::set RemoteImages; - typedef typename librbd::journal::TypeTraits::Journaler Journaler; typedef boost::optional OptionalState; typedef boost::optional @@ -292,7 +266,7 @@ private: ImageDeleter* m_image_deleter; InstanceWatcher *m_instance_watcher; - RemoteImages m_remote_images; + Peers m_peers; RemoteImage m_remote_image; RadosRef m_local; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 5ce59ac25a0..d6c922ac105 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -99,24 +99,15 @@ void InstanceReplayer::shut_down(Context *on_finish) { } template -void InstanceReplayer::add_peer(std::string mirror_uuid, +void InstanceReplayer::add_peer(std::string peer_uuid, librados::IoCtx io_ctx) { - dout(20) << mirror_uuid << dendl; + dout(20) << peer_uuid << dendl; Mutex::Locker locker(m_lock); - auto result = m_peers.insert(Peer(mirror_uuid, io_ctx)).second; + auto result = m_peers.insert(Peer(peer_uuid, io_ctx)).second; assert(result); } -template -void InstanceReplayer::remove_peer(std::string mirror_uuid) { - dout(20) << mirror_uuid << dendl; - - Mutex::Locker locker(m_lock); - auto result = m_peers.erase(Peer(mirror_uuid)); - assert(result > 0); -} - template void InstanceReplayer::release_all(Context *on_finish) { dout(20) << dendl; @@ -152,7 +143,6 @@ void InstanceReplayer::acquire_image(InstanceWatcher *instance_watcher, assert(m_on_shut_down == nullptr); auto it = m_image_replayers.find(global_image_id); - if (it == m_image_replayers.end()) { auto image_replayer = ImageReplayer::create( m_threads, m_image_deleter, instance_watcher, m_local_rados, @@ -163,18 +153,14 @@ void InstanceReplayer::acquire_image(InstanceWatcher *instance_watcher, it = m_image_replayers.insert(std::make_pair(global_image_id, image_replayer)).first; - } - - auto image_replayer = it->second; - if (!peer_mirror_uuid.empty()) { - auto iter = m_peers.find(Peer(peer_mirror_uuid)); - assert(iter != m_peers.end()); - auto io_ctx = iter->io_ctx; - image_replayer->add_remote_image(peer_mirror_uuid, peer_image_id, io_ctx); + // TODO only a single peer is currently supported + assert(m_peers.size() == 1); + auto peer = *m_peers.begin(); + image_replayer->add_peer(peer.peer_uuid, peer.io_ctx); } - start_image_replayer(image_replayer); + start_image_replayer(it->second); m_threads->work_queue->queue(on_finish, 0); } @@ -192,7 +178,6 @@ void InstanceReplayer::release_image(const std::string &global_image_id, assert(m_on_shut_down == nullptr); auto it = m_image_replayers.find(global_image_id); - if (it == m_image_replayers.end()) { dout(20) << global_image_id << ": not found" << dendl; m_threads->work_queue->queue(on_finish, 0); @@ -200,17 +185,6 @@ void InstanceReplayer::release_image(const std::string &global_image_id, } auto image_replayer = it->second; - if (!peer_mirror_uuid.empty()) { - image_replayer->remove_remote_image(peer_mirror_uuid, peer_image_id, - schedule_delete); - } - - if (!image_replayer->remote_images_empty()) { - dout(20) << global_image_id << ": still has peer images" << dendl; - m_threads->work_queue->queue(on_finish, 0); - return; - } - m_image_replayers.erase(it); on_finish = new FunctionContext( @@ -218,17 +192,6 @@ void InstanceReplayer::release_image(const std::string &global_image_id, image_replayer->destroy(); on_finish->complete(0); }); - - if (schedule_delete) { - on_finish = new FunctionContext( - [this, image_replayer, on_finish] (int r) { - auto global_image_id = image_replayer->get_global_image_id(); - m_image_deleter->schedule_image_delete( - m_local_rados, m_local_pool_id, global_image_id, false); - on_finish->complete(0); - }); - } - stop_image_replayer(image_replayer, on_finish); } diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index e85938b71a8..d5c59748a93 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -52,8 +52,7 @@ public: void init(Context *on_finish); void shut_down(Context *on_finish); - void add_peer(std::string mirror_uuid, librados::IoCtx io_ctx); - void remove_peer(std::string mirror_uuid); + void add_peer(std::string peer_uuid, librados::IoCtx io_ctx); void acquire_image(InstanceWatcher *instance_watcher, const std::string &global_image_id, @@ -91,30 +90,6 @@ private: * @endverbatim */ - struct Peer { - std::string mirror_uuid; - librados::IoCtx io_ctx; - - Peer() { - } - - Peer(const std::string &mirror_uuid) : mirror_uuid(mirror_uuid) { - } - - Peer(const std::string &mirror_uuid, librados::IoCtx &io_ctx) - : mirror_uuid(mirror_uuid), io_ctx(io_ctx) { - } - - inline bool operator<(const Peer &rhs) const { - return mirror_uuid < rhs.mirror_uuid; - } - inline bool operator==(const Peer &rhs) const { - return mirror_uuid == rhs.mirror_uuid; - } - }; - - typedef std::set Peers; - Threads *m_threads; ServiceDaemon* m_service_daemon; ImageDeleter* m_image_deleter; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index d40a93a1e2a..4bfd1fc629f 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -620,37 +620,6 @@ void PoolReplayer::handle_update(const std::string &mirror_uuid, m_remote_pool_watcher->get_image_count()); } - std::string removed_remote_peer_id; - ImageIds removed_remote_image_ids; - if (m_initial_mirror_image_ids.find(mirror_uuid) == - m_initial_mirror_image_ids.end() && - m_initial_mirror_image_ids.size() < 2) { - m_initial_mirror_image_ids[mirror_uuid] = added_image_ids; - - if (m_initial_mirror_image_ids.size() == 2) { - dout(10) << "local and remote pools refreshed" << dendl; - - // both local and remote initial pool listing received. derive - // removal notifications for the remote pool - auto &local_image_ids = m_initial_mirror_image_ids.begin()->second; - auto &remote_image_ids = m_initial_mirror_image_ids.rbegin()->second; - removed_remote_peer_id = m_initial_mirror_image_ids.rbegin()->first; - for (auto &local_image_id : local_image_ids) { - if (remote_image_ids.find(local_image_id) == remote_image_ids.end()) { - removed_remote_image_ids.emplace(local_image_id.global_id, ""); - } - } - local_image_ids.clear(); - remote_image_ids.clear(); - } - } - - if (!mirror_uuid.empty() && m_peer.uuid != mirror_uuid) { - m_instance_replayer->remove_peer(m_peer.uuid); - m_instance_replayer->add_peer(mirror_uuid, m_remote_io_ctx); - m_peer.uuid = mirror_uuid; - } - m_update_op_tracker.start_op(); Context *ctx = new FunctionContext([this](int r) { dout(20) << "complete handle_update: r=" << r << dendl; @@ -667,13 +636,10 @@ void PoolReplayer::handle_update(const std::string &mirror_uuid, gather_ctx->new_sub()); } - for (auto &image_id : removed_image_ids) { - // for now always send to myself (the leader) - std::string &instance_id = m_instance_watcher->get_instance_id(); - m_instance_watcher->notify_image_release(instance_id, image_id.global_id, - mirror_uuid, image_id.id, true, - gather_ctx->new_sub()); - if (!mirror_uuid.empty()) { + if (!mirror_uuid.empty()) { + for (auto &image_id : removed_image_ids) { + // for now always send to myself (the leader) + std::string &instance_id = m_instance_watcher->get_instance_id(); m_instance_watcher->notify_peer_image_removed(instance_id, image_id.global_id, mirror_uuid, @@ -681,16 +647,6 @@ void PoolReplayer::handle_update(const std::string &mirror_uuid, } } - // derived removal events for remote after initial image listing - for (auto& image_id : removed_remote_image_ids) { - // for now always send to myself (the leader) - std::string &instance_id = m_instance_watcher->get_instance_id(); - m_instance_watcher->notify_image_release(instance_id, image_id.global_id, - removed_remote_peer_id, - image_id.id, true, - gather_ctx->new_sub()); - } - gather_ctx->activate(); } @@ -718,7 +674,6 @@ void PoolReplayer::init_local_pool_watcher(Context *on_finish) { assert(!m_local_pool_watcher); m_local_pool_watcher.reset(new PoolWatcher<>( m_threads, m_local_io_ctx, m_local_pool_watcher_listener)); - m_initial_mirror_image_ids.clear(); // ensure the initial set of local images is up-to-date // after acquiring the leader role diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 73dd663643f..ca693ef7480 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -137,8 +137,6 @@ private: std::string m_asok_hook_name; AdminSocketHook *m_asok_hook = nullptr; - std::map m_initial_mirror_image_ids; - service_daemon::CalloutId m_callout_id = service_daemon::CALLOUT_ID_NONE; class PoolReplayerThread : public Thread { diff --git a/src/tools/rbd_mirror/types.h b/src/tools/rbd_mirror/types.h index cec8a4deb63..ba8aa7df1fa 100644 --- a/src/tools/rbd_mirror/types.h +++ b/src/tools/rbd_mirror/types.h @@ -10,6 +10,7 @@ #include #include +#include "include/rados/librados.hpp" #include "include/rbd/librbd.hpp" namespace rbd { @@ -41,6 +42,25 @@ std::ostream &operator<<(std::ostream &, const ImageId &image_id); typedef std::set ImageIds; +struct Peer { + std::string peer_uuid; + librados::IoCtx io_ctx; + + Peer() { + } + Peer(const std::string &peer_uuid) : peer_uuid(peer_uuid) { + } + Peer(const std::string &peer_uuid, librados::IoCtx& io_ctx) + : peer_uuid(peer_uuid), io_ctx(io_ctx) { + } + + inline bool operator<(const Peer &rhs) const { + return peer_uuid < rhs.peer_uuid; + } +}; + +typedef std::set Peers; + struct peer_t { peer_t() = default; peer_t(const std::string &uuid, const std::string &cluster_name,