From f3978ba56636df9840d8e8185470ba64c9787fac Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 14 Dec 2017 17:23:49 -0500
Subject: [PATCH] rbd-mirror: only the leader should initialize the image
 deleter

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/tools/rbd_mirror/PoolReplayer.cc | 81 +++++++++++++++++++++-------
 src/tools/rbd_mirror/PoolReplayer.h  |  6 +++
 2 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index cca56443ba5..93054d5b73c 100644
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -311,13 +311,6 @@ void PoolReplayer::init()
 
   dout(20) << "connected to " << m_peer << dendl;
 
-  // TODO
-  C_SaferCond image_deleter_ctx;
-  m_image_deleter.reset(new ImageDeleter<>(m_local_io_ctx, m_threads,
-                                           m_service_daemon));
-  m_image_deleter->init(&image_deleter_ctx);
-  image_deleter_ctx.wait();
-
   m_instance_replayer.reset(InstanceReplayer<>::create(
     m_threads, m_service_daemon, m_local_rados, local_mirror_uuid,
     m_local_pool_id));
@@ -376,13 +369,7 @@ void PoolReplayer::shut_down() {
     m_instance_replayer.reset();
   }
 
-  // TODO
-  if (m_image_deleter) {
-    C_SaferCond image_deleter_ctx;
-    m_image_deleter->shut_down(&image_deleter_ctx);
-    image_deleter_ctx.wait();
-  }
-
+  assert(!m_image_deleter);
   assert(!m_local_pool_watcher);
   assert(!m_remote_pool_watcher);
   m_local_rados.reset();
@@ -566,9 +553,11 @@ void PoolReplayer::print_status(Formatter *f, stringstream *ss)
 
   m_instance_replayer->print_status(f, ss);
 
-  f->open_object_section("image_deleter");
-  m_image_deleter->print_status(f, ss);
-  f->close_section();
+  if (m_image_deleter) {
+    f->open_object_section("image_deleter");
+    m_image_deleter->print_status(f, ss);
+    f->close_section();
+  }
 
   f->close_section();
   f->flush(*ss);
@@ -709,7 +698,7 @@ void PoolReplayer::handle_pre_release_leader(Context *on_finish) {
 
   m_service_daemon->remove_attribute(m_local_pool_id, SERVICE_DAEMON_LEADER_KEY);
   m_instance_watcher->handle_release_leader();
-  shut_down_pool_watchers(on_finish);
+  shut_down_image_deleter(on_finish);
 }
 
 void PoolReplayer::init_local_pool_watcher(Context *on_finish) {
@@ -747,12 +736,66 @@ void PoolReplayer::init_remote_pool_watcher(Context *on_finish) {
   assert(!m_remote_pool_watcher);
   m_remote_pool_watcher.reset(new PoolWatcher<>(
     m_threads, m_remote_io_ctx, m_remote_pool_watcher_listener));
+
+  auto ctx = new FunctionContext([this, on_finish](int r) {
+      handle_init_remote_pool_watcher(r, on_finish);
+    });
   m_remote_pool_watcher->init(create_async_context_callback(
-    m_threads->work_queue, on_finish));
+    m_threads->work_queue, ctx));
+}
+
+void PoolReplayer::handle_init_remote_pool_watcher(int r, Context *on_finish) {
+  dout(20) << "r=" << r << dendl;
+  if (r < 0) {
+    derr << "failed to retrieve remote images: " << cpp_strerror(r) << dendl;
+    on_finish->complete(r);
+    return;
+  }
+
+  init_image_deleter(on_finish);
+}
+
+void PoolReplayer::init_image_deleter(Context *on_finish) {
+  dout(20) << dendl;
 
+  Mutex::Locker locker(m_lock);
+  assert(!m_image_deleter);
+
+  m_image_deleter.reset(new ImageDeleter<>(m_local_io_ctx, m_threads,
+                                           m_service_daemon));
+  m_image_deleter->init(on_finish);
   m_cond.Signal();
 }
 
+void PoolReplayer::shut_down_image_deleter(Context* on_finish) {
+  dout(20) << dendl;
+  {
+    Mutex::Locker locker(m_lock);
+    if (m_image_deleter) {
+      Context *ctx = new FunctionContext([this, on_finish](int r) {
+          handle_shut_down_image_deleter(r, on_finish);
+	});
+      ctx = create_async_context_callback(m_threads->work_queue, ctx);
+
+      m_image_deleter->shut_down(ctx);
+      return;
+    }
+  }
+  shut_down_pool_watchers(on_finish);
+}
+
+void PoolReplayer::handle_shut_down_image_deleter(int r, Context* on_finish) {
+  dout(20) << "r=" << r << dendl;
+
+  {
+    Mutex::Locker locker(m_lock);
+    assert(m_image_deleter);
+    m_image_deleter.reset();
+  }
+
+  shut_down_pool_watchers(on_finish);
+}
+
 void PoolReplayer::shut_down_pool_watchers(Context *on_finish) {
   dout(20) << dendl;
 
diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h
index 1cfa897ca11..c9967700d9d 100644
--- a/src/tools/rbd_mirror/PoolReplayer.h
+++ b/src/tools/rbd_mirror/PoolReplayer.h
@@ -98,6 +98,12 @@ private:
   void handle_init_local_pool_watcher(int r, Context *on_finish);
 
   void init_remote_pool_watcher(Context *on_finish);
+  void handle_init_remote_pool_watcher(int r, Context *on_finish);
+
+  void init_image_deleter(Context* on_finish);
+
+  void shut_down_image_deleter(Context* on_finish);
+  void handle_shut_down_image_deleter(int r, Context* on_finish);
 
   void shut_down_pool_watchers(Context *on_finish);
   void handle_shut_down_pool_watchers(int r, Context *on_finish);
-- 
2.39.5