From fef7456f8bc85f32cbdb28ce81198a9eb7d56d3b Mon Sep 17 00:00:00 2001 From: Ricardo Dias Date: Tue, 5 Apr 2016 09:47:32 +0100 Subject: [PATCH] rbd-mirror: Replayer: bootstrap existing mirrored images Signed-off-by: Ricardo Dias (cherry picked from commit 3c2615aa6da837bfef2365f701f56a34c49b536f) --- src/tools/rbd_mirror/Replayer.cc | 153 ++++++++++++++++++++++++++++--- src/tools/rbd_mirror/Replayer.h | 29 +++++- 2 files changed, 166 insertions(+), 16 deletions(-) diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index 370b527b50789..ffa3e0644e9e9 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -13,6 +13,7 @@ #include "include/stringify.h" #include "cls/rbd/cls_rbd_client.h" #include "librbd/ObjectWatcher.h" +#include "librbd/internal.h" #include "Replayer.h" #include "Threads.h" @@ -26,6 +27,8 @@ using std::string; using std::unique_ptr; using std::vector; +using librbd::cls_client::dir_get_name; + namespace rbd { namespace mirror { @@ -224,9 +227,11 @@ private: Watcher *m_watcher; }; -Replayer::Replayer(Threads *threads, RadosRef local_cluster, - const peer_t &peer, const std::vector &args) : +Replayer::Replayer(Threads *threads, std::shared_ptr image_deleter, + RadosRef local_cluster, const peer_t &peer, + const std::vector &args) : m_threads(threads), + m_image_deleter(image_deleter), m_lock(stringify("rbd::mirror::Replayer ") + stringify(peer)), m_peer(peer), m_args(args), @@ -257,8 +262,6 @@ int Replayer::init() { dout(20) << "replaying for " << m_peer << dendl; - m_image_deleter.reset(new ImageDeleter(m_peer.cluster_name, m_local)); - // NOTE: manually bootstrap a CephContext here instead of via // the librados API to avoid mixing global singletons between // the librados shared library and the daemon @@ -324,6 +327,9 @@ int Replayer::init() dout(20) << "connected to " << m_peer << dendl; + // Bootstrap existing mirroring images + init_local_mirroring_images(); + // TODO: make interval configurable m_pool_watcher.reset(new PoolWatcher(m_remote, 30, m_lock, m_cond)); m_pool_watcher->refresh_images(); @@ -333,6 +339,104 @@ int Replayer::init() return 0; } +void Replayer::init_local_mirroring_images() { + list > pools; + int r = m_local->pool_list2(pools); + if (r < 0) { + derr << "error listing pools: " << cpp_strerror(r) << dendl; + return; + } + + for (auto kv : pools) { + int64_t pool_id = kv.first; + string pool_name = kv.second; + int64_t base_tier; + r = m_local->pool_get_base_tier(pool_id, &base_tier); + if (r == -ENOENT) { + dout(10) << "pool " << pool_name << " no longer exists" << dendl; + continue; + } else if (r < 0) { + derr << "Error retrieving base tier for pool " << pool_name << dendl; + continue; + } + if (pool_id != base_tier) { + // pool is a cache; skip it + continue; + } + + librados::IoCtx ioctx; + r = m_local->ioctx_create2(pool_id, ioctx); + if (r == -ENOENT) { + dout(10) << "pool " << pool_name << " no longer exists" << dendl; + continue; + } else if (r < 0) { + derr << "Error accessing pool " << pool_name << cpp_strerror(r) << dendl; + continue; + } + + rbd_mirror_mode_t mirror_mode; + r = librbd::mirror_mode_get(ioctx, &mirror_mode); + if (r < 0) { + derr << "could not tell whether mirroring was enabled for " << pool_name + << " : " << cpp_strerror(r) << dendl; + continue; + } + if (mirror_mode == RBD_MIRROR_MODE_DISABLED) { + dout(20) << "pool " << pool_name << " has mirroring disabled" << dendl; + continue; + } + + librados::IoCtx remote_ioctx; + r = m_remote->ioctx_create(ioctx.get_pool_name().c_str(), remote_ioctx); + if (r < 0 && r != -ENOENT) { + dout(10) << "Error connecting to remote pool " << ioctx.get_pool_name() + << ": " << cpp_strerror(r) << dendl; + continue; + } else if (r == -ENOENT) { + // remote pool does not exist anymore, we are going to add the images + // with local pool id + pool_id = ioctx.get_id(); + } + else { + pool_id = remote_ioctx.get_id(); + } + + std::set images; + + std::string last_read = ""; + int max_read = 1024; + do { + std::map mirror_images; + r = librbd::cls_client::mirror_image_list(&ioctx, last_read, max_read, + &mirror_images); + if (r < 0) { + derr << "error listing mirrored image directory: " + << cpp_strerror(r) << dendl; + continue; + } + for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { + std::string image_name; + r = dir_get_name(&ioctx, RBD_DIRECTORY, it->first, &image_name); + if (r < 0) { + derr << "error retrieving local image name: " << cpp_strerror(r) + << dendl; + continue; + } + images.insert(InitImageInfo(it->second, ioctx.get_id(), it->first, + image_name)); + } + if (!mirror_images.empty()) { + last_read = mirror_images.rbegin()->first; + } + r = mirror_images.size(); + } while (r == max_read); + + if (!images.empty()) { + m_init_images[pool_id] = std::move(images); + } + } +} + void Replayer::run() { dout(20) << "enter" << dendl; @@ -345,7 +449,6 @@ void Replayer::run() m_cond.WaitInterval(g_ceph_context, m_lock, seconds(30)); } - // Stopping m_image_deleter.reset(); PoolImageIds empty_sources; @@ -379,13 +482,6 @@ void Replayer::print_status(Formatter *f, stringstream *ss) } } - if (f) { - f->close_section(); - f->open_object_section("image_deleter"); - } - - m_image_deleter->print_status(f, ss); - if (f) { f->close_section(); f->close_section(); @@ -480,6 +576,37 @@ void Replayer::set_sources(const PoolImageIds &pool_image_ids) dout(20) << "enter" << dendl; assert(m_lock.is_locked()); + + if (!m_init_images.empty()) { + dout(20) << "m_init_images has images!" << dendl; + for (auto it = m_init_images.begin(); it != m_init_images.end(); ++it) { + int64_t pool_id = it->first; + std::set& images = it->second; + auto remote_pool_it = pool_image_ids.find(pool_id); + if (remote_pool_it != pool_image_ids.end()) { + const std::set& remote_images = remote_pool_it->second; + for (const auto& remote_image : remote_images) { + auto image = images.find(InitImageInfo(remote_image.global_id)); + if (image != images.end()) { + images.erase(image); + } + } + } + } + // the remaining images in m_init_images must be deleted + for (auto it = m_init_images.begin(); it != m_init_images.end(); ++it) { + for (const auto& image : it->second) { + dout(20) << "scheduling the deletion of init image: " + << image.name << dendl; + m_image_deleter->schedule_image_delete(image.pool_id, image.id, + image.name, image.global_id); + } + } + m_init_images.clear(); + } else { + dout(20) << "m_init_images is empty!" << dendl; + } + for (auto it = m_images.begin(); it != m_images.end();) { int64_t pool_id = it->first; auto &pool_images = it->second; @@ -680,7 +807,7 @@ bool Replayer::stop_image_replayer(unique_ptr > &image_replayer) ); image_replayer->stop(ctx); } else { - // TODO: check how long it is stopping and alert if it is too long. + // TODO: checkhow long it is stopping and alert if it is too long. } return false; diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h index 460f9cf83cbdb..ff01fca5269c2 100644 --- a/src/tools/rbd_mirror/Replayer.h +++ b/src/tools/rbd_mirror/Replayer.h @@ -33,8 +33,9 @@ class MirrorStatusWatchCtx; */ class Replayer { public: - Replayer(Threads *threads, RadosRef local_cluster, const peer_t &peer, - const std::vector &args); + Replayer(Threads *threads, std::shared_ptr image_deleter, + RadosRef local_cluster, const peer_t &peer, + const std::vector &args); ~Replayer(); Replayer(const Replayer&) = delete; Replayer& operator=(const Replayer&) = delete; @@ -52,6 +53,7 @@ private: typedef PoolWatcher::ImageIds ImageIds; typedef PoolWatcher::PoolImageIds PoolImageIds; + void init_local_mirroring_images(); void set_sources(const PoolImageIds &pool_image_ids); void start_image_replayer(unique_ptr > &image_replayer, @@ -62,6 +64,7 @@ private: void mirror_image_status_shut_down(int64_t pool_id); Threads *m_threads; + std::shared_ptr m_image_deleter; Mutex m_lock; Cond m_cond; atomic_t m_stopping; @@ -77,7 +80,27 @@ private: std::unique_ptr > > > m_images; std::map > m_status_watchers; ReplayerAdminSocketHook *m_asok_hook; - std::unique_ptr m_image_deleter; + struct InitImageInfo { + std::string global_id; + int64_t pool_id; + std::string id; + std::string name; + + InitImageInfo(const std::string& global_id, int64_t pool_id = 0, + const std::string &id = "", const std::string &name = "") + : global_id(global_id), pool_id(pool_id), id(id), name(name) { + } + + inline bool operator==(const InitImageInfo &rhs) const { + return (global_id == rhs.global_id && pool_id == rhs.pool_id && + id == rhs.id && name == rhs.name); + } + inline bool operator<(const InitImageInfo &rhs) const { + return global_id < rhs.global_id; + } + }; + + std::map > m_init_images; class ReplayerThread : public Thread { Replayer *m_replayer; -- 2.39.5