rbd-mirror: refresh local images after acquiring leader role

The local image id set should be up-to-date when attempting to
determine which images need to be deleted.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc
index 558df3bec7b270c1dd3eebf19c766bd7092f2784..58b53a23ebec4bcd9cbfc5f3d0c538ca5015dbd1 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.cc
+++ b/src/tools/rbd_mirror/Replayer.cc
@@ -21,6 +21,7 @@
 #include "LeaderWatcher.h"
 #include "Replayer.h"
 #include "Threads.h"
+#include "pool_watcher/RefreshImagesRequest.h"
 
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_rbd_mirror
@@ -206,6 +207,20 @@ private:
   Commands commands;
 };
 
+struct Replayer::C_RefreshLocalImages : public Context {
+  Replayer *replayer;
+  Context *on_finish;
+  ImageIds image_ids;
+
+  C_RefreshLocalImages(Replayer *replayer, Context *on_finish)
+    : replayer(replayer), on_finish(on_finish) {
+  }
+
+  void finish(int r) override {
+    replayer->handle_refresh_local_images(r, std::move(image_ids), on_finish);
+  }
+};
+
 Replayer::Replayer(Threads<librbd::ImageCtx> *threads,
                    std::shared_ptr<ImageDeleter> image_deleter,
                    ImageSyncThrottlerRef<> image_sync_throttler,
@@ -293,11 +308,6 @@ int Replayer::init()
 
   dout(20) << "connected to " << m_peer << dendl;
 
-  r = init_local_mirroring_images();
-  if (r < 0) {
-    return r;
-  }
-
   m_leader_watcher.reset(new LeaderWatcher<>(m_threads, m_local_io_ctx,
                                              &m_leader_listener));
   r = m_leader_watcher->init();
@@ -390,61 +400,6 @@ int Replayer::init_rados(const std::string &cluster_name,
   return 0;
 }
 
-int Replayer::init_local_mirroring_images() {
-  dout(20) << dendl;
-
-  rbd_mirror_mode_t mirror_mode;
-  int r = librbd::api::Mirror<>::mode_get(m_local_io_ctx, &mirror_mode);
-  if (r < 0) {
-    derr << "could not tell whether mirroring was enabled for "
-         << m_local_io_ctx.get_pool_name() << ": " << cpp_strerror(r) << dendl;
-    return r;
-  }
-  if (mirror_mode == RBD_MIRROR_MODE_DISABLED) {
-    dout(20) << "pool " << m_local_io_ctx.get_pool_name() << " "
-             << "has mirroring disabled" << dendl;
-    return -ENOENT;
-  }
-
-  ImageIds image_ids;
-
-  std::string last_read = "";
-  int max_read = 1024;
-  do {
-    std::map<std::string, std::string> mirror_images;
-    r = librbd::cls_client::mirror_image_list(&m_local_io_ctx, last_read,
-                                              max_read, &mirror_images);
-    if (r < 0) {
-      derr << "error listing mirrored image directory: "
-           << cpp_strerror(r) << dendl;
-      return r;
-    }
-    for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) {
-      std::string image_name;
-      r = dir_get_name(&m_local_io_ctx, RBD_DIRECTORY, it->first, &image_name);
-      if (r == -ENOENT) {
-        dout(20) << "orphaned mirror image: " << it->first << dendl;
-        continue;
-      } else if (r < 0) {
-        derr << "error retrieving local image name: " << cpp_strerror(r)
-             << dendl;
-        return r;
-      }
-
-      dout(20) << "local image: " << it->second << " (" << it->first << ")"
-               << dendl;
-      image_ids.insert(ImageId(it->second, it->first, image_name));
-    }
-    if (!mirror_images.empty()) {
-      last_read = mirror_images.rbegin()->first;
-    }
-    r = mirror_images.size();
-  } while (r == max_read);
-
-  m_init_image_ids = std::move(image_ids);
-  return 0;
-}
-
 void Replayer::run()
 {
   dout(20) << "enter" << dendl;
@@ -849,6 +804,45 @@ void Replayer::stop_image_replayers(Context *on_finish) {
 
 void Replayer::handle_post_acquire_leader(Context *on_finish) {
   dout(20) << dendl;
+  refresh_local_images(on_finish);
+}
+
+void Replayer::handle_pre_release_leader(Context *on_finish) {
+  dout(20) << dendl;
+  shut_down_pool_watcher(on_finish);
+}
+
+void Replayer::refresh_local_images(Context *on_finish) {
+  dout(20) << dendl;
+
+  // ensure the initial set of local images is up-to-date
+  // after acquiring the leader role
+  auto ctx = new C_RefreshLocalImages(this, on_finish);
+  auto req = pool_watcher::RefreshImagesRequest<>::create(
+    m_local_io_ctx, &ctx->image_ids, ctx);
+  req->send();
+}
+
+void Replayer::handle_refresh_local_images(int r, ImageIds &&image_ids,
+                                           Context *on_finish) {
+  dout(20) << "r=" << r << dendl;
+
+  {
+    Mutex::Locker locker(m_lock);
+    m_init_image_ids = std::move(image_ids);
+  }
+
+  if (r < 0) {
+    derr << "failed to retrieve local images: " << cpp_strerror(r) << dendl;
+    on_finish->complete(r);
+    return;
+  }
+
+  init_pool_watcher(on_finish);
+}
+
+void Replayer::init_pool_watcher(Context *on_finish) {
+  dout(20) << dendl;
 
   Mutex::Locker locker(m_lock);
   assert(!m_pool_watcher);
@@ -860,22 +854,23 @@ void Replayer::handle_post_acquire_leader(Context *on_finish) {
   m_cond.Signal();
 }
 
-void Replayer::handle_pre_release_leader(Context *on_finish) {
-  dout(20) << dendl;
-  shut_down_pool_watcher(on_finish);
-}
-
 void Replayer::shut_down_pool_watcher(Context *on_finish) {
   dout(20) << dendl;
 
-  Context *ctx = new FunctionContext([this, on_finish](int r) {
-      handle_shut_down_pool_watcher(r, on_finish);
-    });
-  ctx = create_async_context_callback(m_threads->work_queue, ctx);
+  {
+    Mutex::Locker locker(m_lock);
+    if (m_pool_watcher) {
+      Context *ctx = new FunctionContext([this, on_finish](int r) {
+          handle_shut_down_pool_watcher(r, on_finish);
+      });
+      ctx = create_async_context_callback(m_threads->work_queue, ctx);
+
+      m_pool_watcher->shut_down(ctx);
+      return;
+    }
+  }
 
-  Mutex::Locker locker(m_lock);
-  assert(m_pool_watcher);
-  m_pool_watcher->shut_down(ctx);
+  on_finish->complete(0);
 }
 
 void Replayer::handle_shut_down_pool_watcher(int r, Context *on_finish) {

diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h
index aff667381a9574ebe23e17eb70e5dd7e6e02faa9..e1da95cc72796f847390326752be866d6ed425b4 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.h
+++ b/src/tools/rbd_mirror/Replayer.h
@@ -71,7 +71,7 @@ private:
     }
   };
 
-  int init_local_mirroring_images();
+  struct C_RefreshLocalImages;
 
   void handle_update(const ImageIds &added_image_ids,
                      const ImageIds &removed_image_ids);
@@ -88,6 +88,11 @@ private:
   void handle_post_acquire_leader(Context *on_finish);
   void handle_pre_release_leader(Context *on_finish);
 
+  void refresh_local_images(Context *on_finish);
+  void handle_refresh_local_images(int r, ImageIds &&image_ids,
+                                   Context *on_finish);
+
+  void init_pool_watcher(Context *on_finish);
   void shut_down_pool_watcher(Context *on_finish);
   void handle_shut_down_pool_watcher(int r, Context *on_finish);