From 7565acc14fb1e0736c3cfaeea985bc63ed2c6ba6 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 28 Nov 2016 13:38:18 -0500 Subject: [PATCH] rbd-mirror: preliminary support to track multiple remote peer image sources Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/test_ImageReplayer.cc | 6 +- .../rbd_mirror/test_mock_ImageReplayer.cc | 6 +- src/tools/rbd_mirror/ImageReplayer.cc | 82 +++++++++++-------- src/tools/rbd_mirror/ImageReplayer.h | 67 +++++++++++---- src/tools/rbd_mirror/Replayer.cc | 12 ++- 5 files changed, 111 insertions(+), 62 deletions(-) diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index e4231f75cc06f..4a0cd12d6c896 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -133,9 +133,9 @@ public: void create_replayer() { m_replayer = new ImageReplayerT(m_threads, m_image_deleter, m_image_sync_throttler, rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), - rbd::mirror::RadosRef(new librados::Rados(m_remote_ioctx)), - m_local_mirror_uuid, m_remote_mirror_uuid, m_local_ioctx.get_id(), - m_remote_pool_id, m_remote_image_id, "global image id"); + m_local_mirror_uuid, m_local_ioctx.get_id(), "global image id"); + m_replayer->add_remote_image(m_remote_mirror_uuid, m_remote_image_id, + m_remote_ioctx); } void start() diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 3503b76be0ef4..4db6ee4f3ee67 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -265,9 +265,9 @@ public: m_image_replayer = new MockImageReplayer( m_threads, m_image_deleter, m_image_sync_throttler, rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), - rbd::mirror::RadosRef(new librados::Rados(m_remote_io_ctx)), - "local_mirror_uuid", "remote_mirror_uuid", m_local_io_ctx.get_id(), - m_remote_io_ctx.get_id(), m_remote_image_ctx->id, "global image id"); + "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id"); + m_image_replayer->add_remote_image( + "remote_mirror_uuid", m_remote_image_ctx->id, m_remote_io_ctx); } virtual void TearDown() { diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 13a78741d51c0..65a14a426c9e8 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -267,27 +267,19 @@ template ImageReplayer::ImageReplayer(Threads *threads, shared_ptr image_deleter, ImageSyncThrottlerRef image_sync_throttler, - RadosRef local, RadosRef remote, - const std::string &local_mirror_uuid, - const std::string &remote_mirror_uuid, - int64_t local_pool_id, - int64_t remote_pool_id, - const std::string &remote_image_id, + RadosRef local, + const std::string &local_mirror_uuid, + int64_t local_pool_id, const std::string &global_image_id) : m_threads(threads), m_image_deleter(image_deleter), m_image_sync_throttler(image_sync_throttler), m_local(local), - m_remote(remote), m_local_mirror_uuid(local_mirror_uuid), - m_remote_mirror_uuid(remote_mirror_uuid), - m_remote_pool_id(remote_pool_id), m_local_pool_id(local_pool_id), - m_remote_image_id(remote_image_id), m_global_image_id(global_image_id), - m_name(stringify(remote_pool_id) + "/" + remote_image_id), - m_lock("rbd::mirror::ImageReplayer " + stringify(remote_pool_id) + " " + - remote_image_id), + m_lock("rbd::mirror::ImageReplayer " + stringify(local_pool_id) + " " + + global_image_id), m_progress_cxt(this), m_journal_listener(new JournalListener(this)), m_remote_listener(this) @@ -297,11 +289,11 @@ ImageReplayer::ImageReplayer(Threads *threads, // re-registered using "remote_pool_name/remote_image_name" name. std::string pool_name; - int r = m_remote->pool_reverse_lookup(m_remote_pool_id, &pool_name); + int r = m_local->pool_reverse_lookup(m_local_pool_id, &pool_name); if (r < 0) { - derr << "error resolving remote pool " << m_remote_pool_id + derr << "error resolving local pool " << m_local_pool_id << ": " << cpp_strerror(r) << dendl; - pool_name = stringify(m_remote_pool_id); + pool_name = stringify(m_local_pool_id); } m_name = pool_name + "/" + m_global_image_id; @@ -327,6 +319,32 @@ ImageReplayer::~ImageReplayer() delete m_asok_hook; } +template +void ImageReplayer::add_remote_image(const std::string &mirror_uuid, + const std::string &image_id, + librados::IoCtx &io_ctx) { + Mutex::Locker locker(m_lock); + + RemoteImage remote_image(mirror_uuid, image_id, io_ctx); + auto it = m_remote_images.find(remote_image); + if (it == m_remote_images.end()) { + m_remote_images.insert(remote_image); + } +} + +template +void ImageReplayer::remove_remote_image(const std::string &mirror_uuid, + const std::string &image_id) { + Mutex::Locker locker(m_lock); + m_remote_images.erase({mirror_uuid, image_id}); +} + +template +bool ImageReplayer::remote_images_empty() const { + Mutex::Locker locker(m_lock); + return m_remote_images.empty(); +} + template void ImageReplayer::set_state_description(int r, const std::string &desc) { dout(20) << r << " " << desc << dendl; @@ -351,12 +369,18 @@ void ImageReplayer::start(Context *on_finish, bool manual) dout(5) << "stopped manually, ignoring start without manual flag" << dendl; r = -EPERM; + } else if (m_remote_images.empty()) { + derr << "no remote images associated with replayer" << dendl; + r = -EINVAL; } else { m_state = STATE_STARTING; m_last_r = 0; m_state_desc.clear(); m_manual_stop = false; + // TODO bootstrap will need to support multiple remote images + m_remote_image = *m_remote_images.begin(); + if (on_finish != nullptr) { assert(m_on_start_finish == nullptr); m_on_start_finish = on_finish; @@ -372,14 +396,6 @@ void ImageReplayer::start(Context *on_finish, bool manual) return; } - r = m_remote->ioctx_create2(m_remote_pool_id, m_remote_ioctx); - if (r < 0) { - derr << "error opening ioctx for remote pool " << m_remote_pool_id - << ": " << cpp_strerror(r) << dendl; - on_start_fail(r, "error opening remote pool"); - return; - } - r = m_local->ioctx_create2(m_local_pool_id, m_local_ioctx); if (r < 0) { derr << "error opening ioctx for local pool " << m_local_pool_id @@ -395,9 +411,10 @@ void ImageReplayer::start(Context *on_finish, bool manual) m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock, m_remote_ioctx, - m_remote_image_id, m_local_mirror_uuid, - settings); + &m_threads->timer_lock, + m_remote_image.io_ctx, + m_remote_image.image_id, + m_local_mirror_uuid, settings); bootstrap(); } @@ -410,10 +427,10 @@ void ImageReplayer::bootstrap() { ImageReplayer, &ImageReplayer::handle_bootstrap>(this); BootstrapRequest *request = BootstrapRequest::create( - m_local_ioctx, m_remote_ioctx, m_image_sync_throttler, - &m_local_image_ctx, m_local_image_name, m_remote_image_id, + m_local_ioctx, m_remote_image.io_ctx, m_image_sync_throttler, + &m_local_image_ctx, m_local_image_name, m_remote_image.image_id, m_global_image_id, m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock, m_local_mirror_uuid, m_remote_mirror_uuid, + &m_threads->timer_lock, m_local_mirror_uuid, m_remote_image.mirror_uuid, m_remote_journaler, &m_client_meta, ctx, &m_do_resync, &m_progress_cxt); { @@ -1008,7 +1025,7 @@ void ImageReplayer::allocate_local_tag() { std::string mirror_uuid = m_replay_tag_data.mirror_uuid; if (mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID || mirror_uuid == m_local_mirror_uuid) { - mirror_uuid = m_remote_mirror_uuid; + mirror_uuid = m_remote_image.mirror_uuid; } else if (mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) { dout(5) << "encountered image demotion: stopping" << dendl; Mutex::Locker locker(m_lock); @@ -1017,7 +1034,7 @@ void ImageReplayer::allocate_local_tag() { librbd::journal::TagPredecessor predecessor(m_replay_tag_data.predecessor); if (predecessor.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID) { - predecessor.mirror_uuid = m_remote_mirror_uuid; + predecessor.mirror_uuid = m_remote_image.mirror_uuid; } else if (predecessor.mirror_uuid == m_local_mirror_uuid) { predecessor.mirror_uuid = librbd::Journal<>::LOCAL_MIRROR_UUID; } @@ -1502,7 +1519,6 @@ void ImageReplayer::handle_shut_down(int r) { dout(20) << "stop complete" << dendl; m_local_ioctx.close(); - m_remote_ioctx.close(); ReplayStatusFormatter::destroy(m_replay_status_formatter); m_replay_status_formatter = nullptr; diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 0964a86e465bd..41c3fac37c9ee 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -23,6 +23,8 @@ #include "ImageDeleter.h" #include "ProgressContext.h" #include "types.h" +#include +#include #include class AdminSocketHook; @@ -69,11 +71,8 @@ public: ImageReplayer(Threads *threads, std::shared_ptr image_deleter, ImageSyncThrottlerRef image_sync_throttler, - RadosRef local, RadosRef remote, - const std::string &local_mirror_uuid, - const std::string &remote_mirror_uuid, int64_t local_pool_id, - int64_t remote_pool_id, const std::string &remote_image_id, - const std::string &global_image_id); + RadosRef local, const std::string &local_mirror_uuid, + int64_t local_pool_id, const std::string &global_image_id); virtual ~ImageReplayer(); ImageReplayer(const ImageReplayer&) = delete; ImageReplayer& operator=(const ImageReplayer&) = delete; @@ -91,18 +90,19 @@ public: return (m_last_r == -EBLACKLISTED); } + void add_remote_image(const std::string &remote_mirror_uuid, + const std::string &remote_image_id, + librados::IoCtx &remote_io_ctx); + void remove_remote_image(const std::string &remote_mirror_uuid, + const std::string &remote_image_id); + bool remote_images_empty() const; + inline int64_t get_local_pool_id() const { return m_local_pool_id; } - inline int64_t get_remote_pool_id() const { - return m_remote_pool_id; - } inline const std::string& get_global_image_id() const { return m_global_image_id; } - inline const std::string& get_remote_image_id() const { - return m_remote_image_id; - } inline std::string get_local_image_id() { Mutex::Locker locker(m_lock); return m_local_image_id; @@ -203,6 +203,37 @@ protected: bool on_replay_interrupted(); private: + struct RemoteImage { + std::string mirror_uuid; + std::string image_id; + librados::IoCtx io_ctx; + + RemoteImage() { + } + RemoteImage(const std::string &mirror_uuid, + const std::string &image_id) + : mirror_uuid(mirror_uuid), image_id(image_id) { + } + RemoteImage(const std::string &mirror_uuid, + const std::string &image_id, + librados::IoCtx &io_ctx) + : mirror_uuid(mirror_uuid), image_id(image_id), io_ctx(io_ctx) { + } + + inline bool operator<(const RemoteImage &rhs) const { + if (mirror_uuid != rhs.mirror_uuid) { + return mirror_uuid < rhs.mirror_uuid; + } else { + return image_id < rhs.image_id; + } + } + inline bool operator==(const RemoteImage &rhs) const { + return (mirror_uuid == rhs.mirror_uuid && image_id == rhs.image_id); + } + }; + + typedef std::set RemoteImages; + typedef typename librbd::journal::TypeTraits::Journaler Journaler; typedef boost::optional OptionalState; @@ -241,11 +272,15 @@ private: Threads *m_threads; std::shared_ptr m_image_deleter; ImageSyncThrottlerRef m_image_sync_throttler; - RadosRef m_local, m_remote; + + RemoteImages m_remote_images; + RemoteImage m_remote_image; + + RadosRef m_local; std::string m_local_mirror_uuid; - std::string m_remote_mirror_uuid; - int64_t m_remote_pool_id, m_local_pool_id; - std::string m_remote_image_id, m_local_image_id, m_global_image_id; + int64_t m_local_pool_id; + std::string m_local_image_id; + std::string m_global_image_id; std::string m_local_image_name; std::string m_name; mutable Mutex m_lock; @@ -257,7 +292,7 @@ private: image_replayer::EventPreprocessor *m_event_preprocessor = nullptr; image_replayer::ReplayStatusFormatter *m_replay_status_formatter = nullptr; - librados::IoCtx m_local_ioctx, m_remote_ioctx; + librados::IoCtx m_local_ioctx; ImageCtxT *m_local_image_ctx = nullptr; decltype(ImageCtxT::journal) m_local_journal = nullptr; diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index 095d24970cd7a..e067e19b0cefe 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -606,8 +606,7 @@ void Replayer::set_sources(const ImageIds &image_ids) for (auto image_it = m_image_replayers.begin(); image_it != m_image_replayers.end();) { auto image_id_it = image_ids.find(image_it->first); - if (image_id_it == image_ids.end() || - image_id_it->id != image_it->second->get_remote_image_id()) { + if (image_id_it == image_ids.end()) { if (image_it->second->is_running()) { dout(20) << "stop image replayer for remote image " << image_it->second->get_global_image_id() << dendl; @@ -647,14 +646,13 @@ void Replayer::set_sources(const ImageIds &image_ids) if (it == m_image_replayers.end()) { unique_ptr > image_replayer(new ImageReplayer<>( m_threads, m_image_deleter, m_image_sync_throttler, m_local_rados, - m_remote_rados, local_mirror_uuid, remote_mirror_uuid, m_local_pool_id, - m_remote_pool_id, image_id.id, image_id.global_id)); + local_mirror_uuid, m_local_pool_id, image_id.global_id)); it = m_image_replayers.insert( std::make_pair(image_id.global_id, std::move(image_replayer))).first; - } else if (image_id.id != it->second->get_remote_image_id()) { - // mismatched replayer in progress of stopping - continue; } + + it->second->add_remote_image(remote_mirror_uuid, image_id.id, + m_remote_io_ctx); if (!it->second->is_running()) { dout(20) << "starting image replayer for remote image " << image_id.global_id << dendl; -- 2.39.5