From: Ricardo Dias Date: Thu, 28 Apr 2016 10:49:43 +0000 (+0100) Subject: rbd-mirror: replayer: Added image-deleter thread to replayer X-Git-Tag: v11.0.0~457^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a6d698829d925894e4afcdec0ee42b1fc1205db0;p=ceph.git rbd-mirror: replayer: Added image-deleter thread to replayer Fixes: http://tracker.ceph.com/issues/14421 Signed-off-by: Ricardo Dias --- diff --git a/src/librbd/internal.h b/src/librbd/internal.h index e33532f9c77f2..71f4d6a998a7f 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -90,6 +90,8 @@ namespace librbd { int snap_set(ImageCtx *ictx, const char *snap_name); + int list_images_v2(librados::IoCtx& io_ctx, + std::map& images); int list(librados::IoCtx& io_ctx, std::vector& names); int list_children(ImageCtx *ictx, std::set > & names); diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 6a10c0e62a413..392eeecd836da 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -394,6 +394,10 @@ void ImageReplayer::handle_bootstrap(int r) { { Mutex::Locker locker(m_lock); m_bootstrap_request = nullptr; + if (m_local_image_ctx) { + m_local_image_id = m_local_image_ctx->id; + m_local_image_name = m_local_image_ctx->name; + } } if (r == -EREMOTEIO) { diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 3dc1287968303..280218350776e 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -88,6 +88,10 @@ public: std::string get_name() { Mutex::Locker l(m_lock); return m_name; }; void set_state_description(int r, const std::string &desc); + inline uint64_t get_local_pool_id() { return m_local_pool_id; } + inline const std::string get_local_image_id() { return m_local_image_id; } + inline const std::string get_global_image_id() { return m_global_image_id; } + inline const std::string get_local_image_name() { return m_local_image_name; } void start(Context *on_finish = nullptr, const BootstrapParams *bootstrap_params = nullptr, diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc index a5bc849acdcca..3177bb7c3cd55 100644 --- a/src/tools/rbd_mirror/Mirror.cc +++ b/src/tools/rbd_mirror/Mirror.cc @@ -345,8 +345,8 @@ void Mirror::update_replayers(const map > &peer_configs) const peer_t &peer = kv.first; if (m_replayers.find(peer) == m_replayers.end()) { dout(20) << "starting replayer for " << peer << dendl; - unique_ptr replayer(new Replayer(m_threads, m_local, peer, - m_args)); + unique_ptr replayer(new Replayer(m_threads, m_image_deleter, + m_local, peer, m_args)); // TODO: make async, and retry connecting within replayer int r = replayer->init(); if (r < 0) { diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 0117cabe11f6b..a525b004401c8 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -103,6 +103,18 @@ void PoolWatcher::refresh_images(bool reschedule) continue; } + std::map images_map; + r = librbd::list_images_v2(ioctx, images_map); + if (r < 0) { + derr << "error retrieving image names from pool " << pool_name << ": " + << cpp_strerror(r) << dendl; + } + + std::map image_id_to_name; + for (const auto& img_pair : images_map) { + image_id_to_name.insert(std::make_pair(img_pair.second, img_pair.first)); + } + std::set image_ids; std::string last_read = ""; int max_read = 1024; @@ -115,7 +127,12 @@ void PoolWatcher::refresh_images(bool reschedule) continue; } for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) { - image_ids.insert(ImageIds(it->first, it->second)); + boost::optional image_name(boost::none); + auto it2 = image_id_to_name.find(it->first); + if (it2 != image_id_to_name.end()) { + image_name = it2->second; + } + image_ids.insert(ImageIds(it->first, image_name, it->second)); } if (!mirror_images.empty()) { last_read = mirror_images.rbegin()->first; diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index 0ab45b41e849a..9eeb1015bbeb5 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -26,14 +26,17 @@ class PoolWatcher { public: struct ImageIds { std::string id; + boost::optional name; std::string global_id; - ImageIds(const std::string &id, const std::string &global_id = "") - : id(id), global_id(global_id) { + ImageIds(const std::string &id, + const boost::optional &name = boost::none, + const std::string &global_id = "") + : id(id), name(name), global_id(global_id) { } inline bool operator==(const ImageIds &rhs) const { - return (id == rhs.id && global_id == rhs.global_id); + return (id == rhs.id && name == rhs.name && global_id == rhs.global_id); } inline bool operator<(const ImageIds &rhs) const { return id < rhs.id; diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index da9aa0e0ede1d..370b527b50789 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -257,6 +257,8 @@ int Replayer::init() { dout(20) << "replaying for " << m_peer << dendl; + m_image_deleter.reset(new ImageDeleter(m_peer.cluster_name, m_local)); + // NOTE: manually bootstrap a CephContext here instead of via // the librados API to avoid mixing global singletons between // the librados shared library and the daemon @@ -344,6 +346,8 @@ void Replayer::run() } // Stopping + m_image_deleter.reset(); + PoolImageIds empty_sources; while (true) { Mutex::Locker l(m_lock); @@ -375,6 +379,13 @@ void Replayer::print_status(Formatter *f, stringstream *ss) } } + if (f) { + f->close_section(); + f->open_object_section("image_deleter"); + } + + m_image_deleter->print_status(f, ss); + if (f) { f->close_section(); f->close_section(); @@ -578,7 +589,7 @@ void Replayer::set_sources(const PoolImageIds &pool_image_ids) dout(20) << "starting image replayer for " << it->second->get_global_image_id() << dendl; } - start_image_replayer(it->second); + start_image_replayer(it->second, image_id.name); } } } @@ -628,13 +639,25 @@ void Replayer::mirror_image_status_shut_down(int64_t pool_id) { m_status_watchers.erase(watcher_it); } -void Replayer::start_image_replayer(unique_ptr > &image_replayer) +void Replayer::start_image_replayer(unique_ptr > &image_replayer, + const boost::optional& image_name) { if (!image_replayer->is_stopped()) { return; } - image_replayer->start(); + if (image_name) { + FunctionContext *ctx = new FunctionContext( + [&] (int r) { + if (r >= 0) { + image_replayer->start(); + } else { + start_image_replayer(image_replayer, image_name); + } + } + ); + m_image_deleter->wait_for_scheduled_deletion(image_name.get(), ctx, false); + } } bool Replayer::stop_image_replayer(unique_ptr > &image_replayer) @@ -644,7 +667,18 @@ bool Replayer::stop_image_replayer(unique_ptr > &image_replayer) } if (image_replayer->is_running()) { - image_replayer->stop(); + FunctionContext *ctx = new FunctionContext( + [&image_replayer, this] (int r) { + if (m_image_deleter) { + m_image_deleter->schedule_image_delete( + image_replayer->get_local_pool_id(), + image_replayer->get_local_image_id(), + image_replayer->get_local_image_name(), + image_replayer->get_global_image_id()); + } + } + ); + image_replayer->stop(ctx); } else { // TODO: check how long it is stopping and alert if it is too long. } diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h index 0dcd5ed72dc75..460f9cf83cbdb 100644 --- a/src/tools/rbd_mirror/Replayer.h +++ b/src/tools/rbd_mirror/Replayer.h @@ -18,6 +18,7 @@ #include "ClusterWatcher.h" #include "ImageReplayer.h" #include "PoolWatcher.h" +#include "ImageDeleter.h" #include "types.h" namespace rbd { @@ -53,7 +54,8 @@ private: void set_sources(const PoolImageIds &pool_image_ids); - void start_image_replayer(unique_ptr > &image_replayer); + void start_image_replayer(unique_ptr > &image_replayer, + const boost::optional& image_name); bool stop_image_replayer(unique_ptr > &image_replayer); int mirror_image_status_init(int64_t pool_id, librados::IoCtx& ioctx); @@ -75,6 +77,7 @@ private: std::unique_ptr > > > m_images; std::map > m_status_watchers; ReplayerAdminSocketHook *m_asok_hook; + std::unique_ptr m_image_deleter; class ReplayerThread : public Thread { Replayer *m_replayer;