From: Jason Dillaman Date: Thu, 24 Mar 2016 13:40:58 +0000 (-0400) Subject: rbd-mirror: bootstrap requires the global image id X-Git-Tag: v10.1.1~64^2~18 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=347349c3f5994215660b92873bdd2ab1b792c69e;p=ceph.git rbd-mirror: bootstrap requires the global image id The global image id can be used to cross-reference images that have been replicated amongst peers. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replay.cc b/src/test/rbd_mirror/image_replay.cc index a51a200d8e3f..bde46e2cc4f5 100644 --- a/src/test/rbd_mirror/image_replay.cc +++ b/src/test/rbd_mirror/image_replay.cc @@ -186,7 +186,8 @@ int main(int argc, const char **argv) local->cct())); replayer = new rbd::mirror::ImageReplayer<>(threads, local, remote, client_id, local_pool_id, remote_pool_id, - remote_image_id); + remote_image_id, + "global image id"); replayer->start(&start_cond, &bootstap_params); r = start_cond.wait(); diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index a4d597319aff..d5de02843612 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -116,7 +116,8 @@ public: m_replayer = new ImageReplayerT(m_threads, rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), rbd::mirror::RadosRef(new librados::Rados(m_remote_ioctx)), - m_client_id, m_local_ioctx.get_id(), m_remote_pool_id, m_remote_image_id); + m_client_id, m_local_ioctx.get_id(), m_remote_pool_id, m_remote_image_id, + "global image id"); } void start(rbd::mirror::ImageReplayer<>::BootstrapParams *bootstap_params = @@ -531,10 +532,11 @@ public: ImageReplayer(rbd::mirror::Threads *threads, rbd::mirror::RadosRef local, rbd::mirror::RadosRef remote, const std::string &client_id, int64_t local_pool_id, - int64_t remote_pool_id, const std::string &remote_image_id) + int64_t remote_pool_id, const std::string &remote_image_id, + const std::string &global_image_id) : rbd::mirror::ImageReplayer<>(threads, local, remote, client_id, local_pool_id, remote_pool_id, - remote_image_id) + remote_image_id, global_image_id) {} void set_error(const std::string &state, int r) { diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc index 5cb68058b227..b8ff3117b30d 100644 --- a/src/test/rbd_mirror/test_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -140,8 +140,14 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), librbd::RBD rbd; rbd.open(ioctx, image, name.c_str()); image.mirror_image_enable(); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); image.close(); - m_mirrored_images[ioctx.get_id()].insert(get_image_id(&ioctx, name)); + + m_mirrored_images[ioctx.get_id()].insert(PoolWatcher::ImageIds( + get_image_id(&ioctx, name), mirror_image_info.global_id)); } if (image_name != nullptr) *image_name = name; @@ -179,8 +185,14 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), librbd::RBD rbd; rbd.open(cioctx, image, name.c_str()); image.mirror_image_enable(); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); image.close(); - m_mirrored_images[cioctx.get_id()].insert(get_image_id(&cioctx, name)); + + m_mirrored_images[cioctx.get_id()].insert(PoolWatcher::ImageIds( + get_image_id(&cioctx, name), mirror_image_info.global_id)); } if (image_name != nullptr) *image_name = name; @@ -198,7 +210,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"), unique_ptr m_pool_watcher; set m_pools; - std::map > m_mirrored_images; + PoolWatcher::PoolImageIds m_mirrored_images; uint64_t m_image_number; uint64_t m_snap_number; diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 9b78c070c7a1..fc1f16b39d49 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -58,6 +58,7 @@ struct BootstrapRequest { librbd::MockImageReplayerImageCtx **local_image_ctx, const std::string &local_image_name, const std::string &remote_image_id, + const std::string &global_image_id, ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid, diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 25c84838c6a9..07334f979895 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -176,7 +176,8 @@ ImageReplayer::ImageReplayer(Threads *threads, RadosRef local, RadosRef remot const std::string &mirror_uuid, int64_t local_pool_id, int64_t remote_pool_id, - const std::string &remote_image_id) : + const std::string &remote_image_id, + const std::string &global_image_id) : m_threads(threads), m_local(local), m_remote(remote), @@ -184,6 +185,7 @@ ImageReplayer::ImageReplayer(Threads *threads, RadosRef local, RadosRef remot m_remote_pool_id(remote_pool_id), m_local_pool_id(local_pool_id), m_remote_image_id(remote_image_id), + m_global_image_id(global_image_id), m_name(stringify(remote_pool_id) + "/" + remote_image_id), m_lock("rbd::mirror::ImageReplayer " + stringify(remote_pool_id) + " " + remote_image_id), @@ -266,9 +268,9 @@ void ImageReplayer::bootstrap() { ImageReplayer, &ImageReplayer::handle_bootstrap>(this); BootstrapRequest *request = BootstrapRequest::create( m_local_ioctx, m_remote_ioctx, &m_local_image_ctx, - m_local_image_name, m_remote_image_id, m_threads->work_queue, - m_threads->timer, &m_threads->timer_lock, m_mirror_uuid, m_remote_journaler, - &m_client_meta, ctx); + m_local_image_name, m_remote_image_id, m_global_image_id, + m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, + m_mirror_uuid, m_remote_journaler, &m_client_meta, ctx); request->send(); } diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 8f36fd099f20..ffff469954d4 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -68,7 +68,8 @@ public: ImageReplayer(Threads *threads, RadosRef local, RadosRef remote, const std::string &mirror_uuid, int64_t local_pool_id, - int64_t remote_pool_id, const std::string &remote_image_id); + int64_t remote_pool_id, const std::string &remote_image_id, + const std::string &global_image_id); virtual ~ImageReplayer(); ImageReplayer(const ImageReplayer&) = delete; ImageReplayer& operator=(const ImageReplayer&) = delete; @@ -160,7 +161,7 @@ private: RadosRef m_local, m_remote; std::string m_mirror_uuid; int64_t m_remote_pool_id, m_local_pool_id; - std::string m_remote_image_id, m_local_image_id; + std::string m_remote_image_id, m_local_image_id, m_global_image_id; std::string m_local_image_name; std::string m_name; Mutex m_lock; diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 2e147f7f34d2..0117cabe11f6 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -17,8 +17,6 @@ #define dout_prefix *_dout << "rbd-mirror: PoolWatcher::" << __func__ << ": " using std::list; -using std::map; -using std::set; using std::string; using std::unique_ptr; using std::vector; @@ -49,7 +47,7 @@ PoolWatcher::~PoolWatcher() m_timer.shutdown(); } -const map >& PoolWatcher::get_images() const +const PoolWatcher::PoolImageIds& PoolWatcher::get_images() const { assert(m_lock.is_locked()); return m_images; @@ -58,7 +56,7 @@ const map >& PoolWatcher::get_images() const void PoolWatcher::refresh_images(bool reschedule) { dout(20) << "enter" << dendl; - map > images; + PoolImageIds images; list > pools; int r = m_cluster->pool_list2(pools); if (r < 0) { @@ -105,7 +103,7 @@ void PoolWatcher::refresh_images(bool reschedule) continue; } - std::set image_ids; + std::set image_ids; std::string last_read = ""; int max_read = 1024; do { @@ -117,7 +115,7 @@ void PoolWatcher::refresh_images(bool reschedule) continue; } for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { - image_ids.insert(it->first); + image_ids.insert(ImageIds(it->first, it->second)); } if (!mirror_images.empty()) { last_read = mirror_images.rbegin()->first; diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index 1358539b7262..0ab45b41e849 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -24,12 +24,30 @@ namespace mirror { */ class PoolWatcher { public: + struct ImageIds { + std::string id; + std::string global_id; + + ImageIds(const std::string &id, const std::string &global_id = "") + : id(id), global_id(global_id) { + } + + inline bool operator==(const ImageIds &rhs) const { + return (id == rhs.id && global_id == rhs.global_id); + } + inline bool operator<(const ImageIds &rhs) const { + return id < rhs.id; + } + }; + typedef std::map > PoolImageIds; + PoolWatcher(RadosRef cluster, double interval_seconds, Mutex &lock, Cond &cond); ~PoolWatcher(); PoolWatcher(const PoolWatcher&) = delete; PoolWatcher& operator=(const PoolWatcher&) = delete; - const std::map >& get_images() const; + + const PoolImageIds& get_images() const; void refresh_images(bool reschedule=true); private: @@ -40,8 +58,8 @@ private: RadosRef m_cluster; SafeTimer m_timer; double m_interval; - // pool id -> image id - std::map > m_images; + + PoolImageIds m_images; }; } // namespace mirror diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index 5aaf18da6ec4..7935d2691621 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -203,7 +203,7 @@ void Replayer::run() } // Stopping - map > empty_sources; + PoolImageIds empty_sources; while (true) { Mutex::Locker l(m_lock); set_sources(empty_sources); @@ -260,7 +260,7 @@ void Replayer::flush() } } -void Replayer::set_sources(const map > &images) +void Replayer::set_sources(const PoolImageIds &pool_image_ids) { dout(20) << "enter" << dendl; @@ -268,7 +268,9 @@ void Replayer::set_sources(const map > &images) for (auto it = m_images.begin(); it != m_images.end();) { int64_t pool_id = it->first; auto &pool_images = it->second; - if (images.find(pool_id) == images.end()) { + + // pool has no mirrored images + if (pool_image_ids.find(pool_id) == pool_image_ids.end()) { for (auto images_it = pool_images.begin(); images_it != pool_images.end();) { if (stop_image_replayer(images_it->second)) { @@ -280,10 +282,12 @@ void Replayer::set_sources(const map > &images) } continue; } + + // shut down replayers for non-mirrored images for (auto images_it = pool_images.begin(); images_it != pool_images.end();) { - if (images.at(pool_id).find(images_it->first) == - images.at(pool_id).end()) { + auto &image_ids = pool_image_ids.at(pool_id); + if (image_ids.find(ImageIds(images_it->first)) == image_ids.end()) { if (stop_image_replayer(images_it->second)) { pool_images.erase(images_it++); } @@ -294,7 +298,8 @@ void Replayer::set_sources(const map > &images) ++it; } - for (const auto &kv : images) { + // (re)start new image replayers + for (const auto &kv : pool_image_ids) { int64_t pool_id = kv.first; // TODO: clean up once remote peer -> image replayer refactored @@ -325,13 +330,13 @@ void Replayer::set_sources(const map > &images) // create entry for pool if it doesn't exist auto &pool_replayers = m_images[pool_id]; for (const auto &image_id : kv.second) { - auto it = pool_replayers.find(image_id); + auto it = pool_replayers.find(image_id.id); if (it == pool_replayers.end()) { unique_ptr > image_replayer(new ImageReplayer<>( m_threads, m_local, m_remote, mirror_uuid, local_ioctx.get_id(), - pool_id, image_id)); + pool_id, image_id.id, image_id.global_id)); it = pool_replayers.insert( - std::make_pair(image_id, std::move(image_replayer))).first; + std::make_pair(image_id.id, std::move(image_replayer))).first; } start_image_replayer(it->second); } diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h index d818f7298364..f7c623b592f2 100644 --- a/src/tools/rbd_mirror/Replayer.h +++ b/src/tools/rbd_mirror/Replayer.h @@ -44,7 +44,10 @@ public: void flush(); private: - void set_sources(const std::map > &images); + typedef PoolWatcher::ImageIds ImageIds; + typedef PoolWatcher::PoolImageIds PoolImageIds; + + void set_sources(const PoolImageIds &pool_image_ids); void start_image_replayer(unique_ptr > &image_replayer); bool stop_image_replayer(unique_ptr > &image_replayer); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 5b112222c1f9..acb39f00cd5c 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -32,15 +32,18 @@ namespace { template struct C_CreateImage : public Context { librados::IoCtx &local_io_ctx; + std::string global_image_id; std::string local_image_name; I *remote_image_ctx; Context *on_finish; C_CreateImage(librados::IoCtx &local_io_ctx, + const std::string &global_image_id, const std::string &local_image_name, I *remote_image_ctx, Context *on_finish) - : local_io_ctx(local_io_ctx), local_image_name(local_image_name), - remote_image_ctx(remote_image_ctx), on_finish(on_finish) { + : local_io_ctx(local_io_ctx), global_image_id(global_image_id), + local_image_name(local_image_name), remote_image_ctx(remote_image_ctx), + on_finish(on_finish) { } virtual void finish(int r) override { @@ -62,7 +65,7 @@ struct C_CreateImage : public Context { remote_image_ctx->stripe_unit, remote_image_ctx->stripe_count, journal_order, journal_splay_width, journal_pool, - "global-image-id"); + global_image_id); on_finish->complete(r); } }; @@ -75,6 +78,7 @@ BootstrapRequest::BootstrapRequest(librados::IoCtx &local_io_ctx, I **local_image_ctx, const std::string &local_image_name, const std::string &remote_image_id, + const std::string &global_image_id, ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid, @@ -83,9 +87,10 @@ BootstrapRequest::BootstrapRequest(librados::IoCtx &local_io_ctx, Context *on_finish) : m_local_io_ctx(local_io_ctx), m_remote_io_ctx(remote_io_ctx), m_local_image_ctx(local_image_ctx), m_local_image_name(local_image_name), - m_remote_image_id(remote_image_id), m_work_queue(work_queue), - m_timer(timer), m_timer_lock(timer_lock), m_mirror_uuid(mirror_uuid), - m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish) { + m_remote_image_id(remote_image_id), m_global_image_id(global_image_id), + m_work_queue(work_queue), m_timer(timer), m_timer_lock(timer_lock), + m_mirror_uuid(mirror_uuid), m_journaler(journaler), + m_client_meta(client_meta), m_on_finish(on_finish) { } template @@ -257,7 +262,8 @@ void BootstrapRequest::create_local_image() { Context *ctx = create_context_callback< BootstrapRequest, &BootstrapRequest::handle_create_local_image>( this); - m_work_queue->queue(new C_CreateImage(m_local_io_ctx, m_local_image_name, + m_work_queue->queue(new C_CreateImage(m_local_io_ctx, m_global_image_id, + m_local_image_name, m_remote_image_ctx, ctx), 0); } diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index 955195161305..b0822be1e758 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -34,6 +34,7 @@ public: ImageCtxT **local_image_ctx, const std::string &local_image_name, const std::string &remote_image_id, + const std::string &global_image_id, ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid, @@ -41,16 +42,17 @@ public: MirrorPeerClientMeta *client_meta, Context *on_finish) { return new BootstrapRequest(local_io_ctx, remote_io_ctx, local_image_ctx, - local_image_name, remote_image_id, work_queue, - timer, timer_lock, mirror_uuid, journaler, - client_meta, on_finish); + local_image_name, remote_image_id, + global_image_id, work_queue, timer, timer_lock, + mirror_uuid, journaler, client_meta, on_finish); } BootstrapRequest(librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, ImageCtxT **local_image_ctx, const std::string &local_image_name, - const std::string &remote_image_id, ContextWQ *work_queue, + const std::string &remote_image_id, + const std::string &global_image_id, ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid, Journaler *journaler, MirrorPeerClientMeta *client_meta, Context *on_finish); @@ -110,6 +112,7 @@ private: std::string m_local_image_name; std::string m_local_image_id; std::string m_remote_image_id; + std::string m_global_image_id; ContextWQ *m_work_queue; SafeTimer *m_timer; Mutex *m_timer_lock;