rbd-mirror: new 'peer image removed' notification to instance watchers

Note: this commit intentionally breaks backwards compatibility with
Luminous RC instance watcher notifications.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
index bdd56bfc39dfc3b85fffec9699972f552a083cab..b9d869fdc9d488a5cbf303fc761a8c6896267747 100644 (file)
--- a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
@@ -81,6 +81,8 @@ struct InstanceReplayer<librbd::MockTestImageCtx> {
                                    const std::string &, Context *));
   MOCK_METHOD5(release_image, void(const std::string &, const std::string &,
                                    const std::string &, bool, Context *));
+  MOCK_METHOD3(remove_peer_image, void(const std::string&, const std::string&,
+                                       Context *));
 };
 
 template <>
@@ -418,6 +420,75 @@ TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) {
   delete instance_watcher2;
 }
 
+TEST_F(TestMockInstanceWatcher, PeerImageRemoved) {
+  MockManagedLock mock_managed_lock;
+
+  librados::IoCtx& io_ctx1 = m_local_io_ctx;
+  std::string instance_id1 = m_instance_id;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
+  MockInstanceReplayer mock_instance_replayer1;
+  auto instance_watcher1 = MockInstanceWatcher::create(
+      io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1);
+
+  librados::Rados cluster;
+  librados::IoCtx io_ctx2;
+  EXPECT_EQ("", connect_cluster_pp(cluster));
+  EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2));
+  std::string instance_id2 = stringify(io_ctx2.get_instance_id());
+  librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
+  MockInstanceReplayer mock_instance_replayer2;
+  auto instance_watcher2 = MockInstanceWatcher::create(
+    io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2);
+
+  InSequence seq;
+
+  // Init instance watcher 1
+  expect_register_instance(mock_io_ctx1, 0);
+  expect_register_watch(mock_io_ctx1, instance_id1);
+  expect_acquire_lock(mock_managed_lock, 0);
+  ASSERT_EQ(0, instance_watcher1->init());
+
+  // Init instance watcher 2
+  expect_register_instance(mock_io_ctx2, 0);
+  expect_register_watch(mock_io_ctx2, instance_id2);
+  expect_acquire_lock(mock_managed_lock, 0);
+  ASSERT_EQ(0, instance_watcher2->init());
+
+  // Peer Image Removed on the same instance
+  EXPECT_CALL(mock_instance_replayer1, remove_peer_image("gid", "uuid", _))
+      .WillOnce(WithArg<2>(CompleteContext(0)));
+  C_SaferCond on_removed1;
+  instance_watcher1->notify_peer_image_removed(instance_id1, "gid", "uuid",
+                                               &on_removed1);
+  ASSERT_EQ(0, on_removed1.wait());
+
+  // Peer Image Removed on the other instance
+  EXPECT_CALL(mock_instance_replayer2, remove_peer_image("gid", "uuid", _))
+      .WillOnce(WithArg<2>(CompleteContext(0)));
+  C_SaferCond on_removed2;
+  instance_watcher1->notify_peer_image_removed(instance_id2, "gid", "uuid",
+                                               &on_removed2);
+  ASSERT_EQ(0, on_removed2.wait());
+
+  // Shutdown instance watcher 1
+  expect_release_lock(mock_managed_lock, 0);
+  expect_unregister_watch(mock_io_ctx1);
+  expect_unregister_instance(mock_io_ctx1, 0);
+  instance_watcher1->shut_down();
+
+  expect_destroy_lock(mock_managed_lock);
+  delete instance_watcher1;
+
+  // Shutdown instance watcher 2
+  expect_release_lock(mock_managed_lock, 0);
+  expect_unregister_watch(mock_io_ctx2);
+  expect_unregister_instance(mock_io_ctx2, 0);
+  instance_watcher2->shut_down();
+
+  expect_destroy_lock(mock_managed_lock);
+  delete instance_watcher2;
+}
+
 TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) {
   MockManagedLock mock_managed_lock;
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
@@ -486,6 +557,53 @@ TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) {
   delete instance_watcher;
 }
 
+TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) {
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+
+  auto instance_watcher = new MockInstanceWatcher(
+    m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+  InSequence seq;
+
+  // Init
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  expect_acquire_lock(mock_managed_lock, 0);
+  ASSERT_EQ(0, instance_watcher->init());
+
+  // Send Acquire Image and cancel
+  EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _))
+    .WillOnce(Invoke(
+                  [this, instance_watcher, &mock_io_ctx](
+                    const std::string& o, librados::AioCompletionImpl *c,
+                    bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) {
+                    c->get();
+                    auto ctx = new FunctionContext(
+                      [instance_watcher, &mock_io_ctx, c, pbl](int r) {
+                        instance_watcher->cancel_notify_requests("other");
+                        ::encode(librbd::watcher::NotifyResponse(), *pbl);
+                        mock_io_ctx.get_mock_rados_client()->
+                            finish_aio_completion(c, -ETIMEDOUT);
+                      });
+                    m_threads->work_queue->queue(ctx, 0);
+                  }));
+
+  C_SaferCond on_acquire;
+  instance_watcher->notify_peer_image_removed("other", "gid", "uuid",
+                                              &on_acquire);
+  ASSERT_EQ(-ECANCELED, on_acquire.wait());
+
+  // Shutdown
+  expect_release_lock(mock_managed_lock, 0);
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, 0);
+  instance_watcher->shut_down();
+
+  expect_destroy_lock(mock_managed_lock);
+  delete instance_watcher;
+}
+
+
 class TestMockInstanceWatcher_NotifySync : public TestMockInstanceWatcher {
 public:
   typedef ImageSyncThrottler<librbd::MockTestImageCtx> MockImageSyncThrottler;

diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index 40a8c1b43205a6f7d6e79043ff4437050ee9c619..5ce59ac25a007a917c6080453a207f7a0d32111b 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -232,6 +232,17 @@ void InstanceReplayer<I>::release_image(const std::string &global_image_id,
   stop_image_replayer(image_replayer, on_finish);
 }
 
+template <typename I>
+void InstanceReplayer<I>::remove_peer_image(const std::string &global_image_id,
+                                            const std::string &peer_mirror_uuid,
+                                            Context *on_finish) {
+  dout(20) << "global_image_id=" << global_image_id << ", "
+           << "peer_mirror_uuid=" << peer_mirror_uuid << dendl;
+
+  // TODO
+  m_threads->work_queue->queue(on_finish, 0);
+}
+
 template <typename I>
 void InstanceReplayer<I>::print_status(Formatter *f, stringstream *ss) {
   dout(20) << dendl;

diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h
index 16618b84123daced34494a214f0ad46e8a039328..e85938b71a895b8fe19ea4ad8c060e4ca0d7b2e2 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.h
+++ b/src/tools/rbd_mirror/InstanceReplayer.h
@@ -64,6 +64,10 @@ public:
                      const std::string &peer_mirror_uuid,
                      const std::string &peer_image_id,
                      bool schedule_delete, Context *on_finish);
+  void remove_peer_image(const std::string &global_image_id,
+                         const std::string &peer_mirror_uuid,
+                         Context *on_finish);
+
   void release_all(Context *on_finish);
 
   void print_status(Formatter *f, stringstream *ss);

diff --git a/src/tools/rbd_mirror/InstanceWatcher.cc b/src/tools/rbd_mirror/InstanceWatcher.cc
index 04e0a5e342d40721510649fd806f6363cc482096..69b860a906d6e918cabfe0f5daadb2fc3fdf35ca 100644 (file)
--- a/src/tools/rbd_mirror/InstanceWatcher.cc
+++ b/src/tools/rbd_mirror/InstanceWatcher.cc
@@ -156,7 +156,7 @@ struct InstanceWatcher<I>::C_NotifyInstanceRequest : public Context {
     }
 
     dout(20) << "C_NotifyInstanceRequest: " << this << " " << __func__
-             << ": sendding to " << instance_id << dendl;
+             << ": sending to " << instance_id << dendl;
     notifier->notify(bl, &response, this);
   }
 
@@ -447,6 +447,30 @@ void InstanceWatcher<I>::notify_image_release(
   }
 }
 
+template <typename I>
+void InstanceWatcher<I>::notify_peer_image_removed(
+    const std::string &instance_id, const std::string &global_image_id,
+    const std::string &peer_mirror_uuid, Context *on_notify_ack) {
+  dout(20) << "instance_id=" << instance_id << ", "
+           << "global_image_id=" << global_image_id << ", "
+           << "peer_mirror_uuid=" << peer_mirror_uuid << dendl;
+
+  Mutex::Locker locker(m_lock);
+  assert(m_on_finish == nullptr);
+
+  if (instance_id == m_instance_id) {
+    handle_peer_image_removed(global_image_id, peer_mirror_uuid, on_notify_ack);
+  } else {
+    uint64_t request_id = ++m_request_seq;
+    bufferlist bl;
+    ::encode(NotifyMessage{PeerImageRemovedPayload{request_id, global_image_id,
+                                                   peer_mirror_uuid}}, bl);
+    auto req = new C_NotifyInstanceRequest(this, instance_id, request_id,
+                                           std::move(bl), on_notify_ack);
+    req->send();
+  }
+}
+
 template <typename I>
 void InstanceWatcher<I>::notify_sync_request(const std::string &sync_id,
                                              Context *on_sync_start) {
@@ -1137,6 +1161,24 @@ void InstanceWatcher<I>::handle_image_release(
   m_work_queue->queue(ctx, 0);
 }
 
+template <typename I>
+void InstanceWatcher<I>::handle_peer_image_removed(
+    const std::string &global_image_id, const std::string &peer_mirror_uuid,
+    Context *on_finish) {
+  dout(20) << "global_image_id=" << global_image_id << ", "
+           << "peer_mirror_uuid=" << peer_mirror_uuid << dendl;
+
+  auto ctx = new FunctionContext(
+      [this, peer_mirror_uuid, global_image_id, on_finish] (int r) {
+        m_instance_replayer->remove_peer_image(global_image_id,
+                                               peer_mirror_uuid, on_finish);
+        m_notify_op_tracker.finish_op();
+      });
+
+  m_notify_op_tracker.start_op();
+  m_work_queue->queue(ctx, 0);
+}
+
 template <typename I>
 void InstanceWatcher<I>::handle_sync_request(const std::string &instance_id,
                                              const std::string &sync_id,
@@ -1220,6 +1262,21 @@ void InstanceWatcher<I>::handle_payload(const std::string &instance_id,
   }
 }
 
+template <typename I>
+void InstanceWatcher<I>::handle_payload(const std::string &instance_id,
+                                        const PeerImageRemovedPayload &payload,
+                                        C_NotifyAck *on_notify_ack) {
+  dout(20) << "remove_peer_image: instance_id=" << instance_id << ", "
+           << "request_id=" << payload.request_id << dendl;
+
+  auto on_finish = prepare_request(instance_id, payload.request_id,
+                                   on_notify_ack);
+  if (on_finish != nullptr) {
+    handle_peer_image_removed(payload.global_image_id, payload.peer_mirror_uuid,
+                              on_finish);
+  }
+}
+
 template <typename I>
 void InstanceWatcher<I>::handle_payload(const std::string &instance_id,
                                         const SyncRequestPayload &payload,

diff --git a/src/tools/rbd_mirror/InstanceWatcher.h b/src/tools/rbd_mirror/InstanceWatcher.h
index 79c2d1c0714a295d8b28f5bb57298ba3ad04fbd4..e04a358df0c191b577ea34eec7a4a20ea6f6e788 100644 (file)
--- a/src/tools/rbd_mirror/InstanceWatcher.h
+++ b/src/tools/rbd_mirror/InstanceWatcher.h
@@ -73,6 +73,10 @@ public:
                             const std::string &peer_mirror_uuid,
                             const std::string &peer_image_id,
                            bool schedule_delete, Context *on_notify_ack);
+  void notify_peer_image_removed(const std::string &instance_id,
+                                 const std::string &global_image_id,
+                                 const std::string &peer_mirror_uuid,
+                                 Context *on_notify_ack);
 
   void notify_sync_request(const std::string &sync_id, Context *on_sync_start);
   bool cancel_sync_request(const std::string &sync_id);
@@ -232,6 +236,9 @@ private:
                             const std::string &peer_mirror_uuid,
                             const std::string &peer_image_id,
                             bool schedule_delete, Context *on_finish);
+  void handle_peer_image_removed(const std::string &global_image_id,
+                                 const std::string &peer_mirror_uuid,
+                                 Context *on_finish);
 
   void handle_sync_request(const std::string &instance_id,
                            const std::string &sync_id, Context *on_finish);
@@ -244,6 +251,9 @@ private:
   void handle_payload(const std::string &instance_id,
                       const instance_watcher::ImageReleasePayload &payload,
                       C_NotifyAck *on_notify_ack);
+  void handle_payload(const std::string &instance_id,
+                      const instance_watcher::PeerImageRemovedPayload &payload,
+                      C_NotifyAck *on_notify_ack);
   void handle_payload(const std::string &instance_id,
                       const instance_watcher::SyncRequestPayload &payload,
                       C_NotifyAck *on_notify_ack);

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index 34b679a83f87f0cc66721a9f68330d036a42a253..d40a93a1e2aefb2f2b6e75ca11aa68fa7ef1b73c 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -673,6 +673,12 @@ void PoolReplayer::handle_update(const std::string &mirror_uuid,
     m_instance_watcher->notify_image_release(instance_id, image_id.global_id,
                                              mirror_uuid, image_id.id, true,
                                              gather_ctx->new_sub());
+    if (!mirror_uuid.empty()) {
+      m_instance_watcher->notify_peer_image_removed(instance_id,
+                                                    image_id.global_id,
+                                                    mirror_uuid,
+                                                    gather_ctx->new_sub());
+    }
   }
 
   // derived removal events for remote after initial image listing

diff --git a/src/tools/rbd_mirror/instance_watcher/Types.cc b/src/tools/rbd_mirror/instance_watcher/Types.cc
index 83ab5d84f8f5761c29bd7dd57b8bea173af540b9..100096e6348a1346b6b73a286d0851010fcf9f16 100644 (file)
--- a/src/tools/rbd_mirror/instance_watcher/Types.cc
+++ b/src/tools/rbd_mirror/instance_watcher/Types.cc
@@ -106,6 +106,24 @@ void ImageReleasePayload::dump(Formatter *f) const {
   f->dump_bool("schedule_delete", schedule_delete);
 }
 
+void PeerImageRemovedPayload::encode(bufferlist &bl) const {
+  PayloadBase::encode(bl);
+  ::encode(global_image_id, bl);
+  ::encode(peer_mirror_uuid, bl);
+}
+
+void PeerImageRemovedPayload::decode(__u8 version, bufferlist::iterator &iter) {
+  PayloadBase::decode(version, iter);
+  ::decode(global_image_id, iter);
+  ::decode(peer_mirror_uuid, iter);
+}
+
+void PeerImageRemovedPayload::dump(Formatter *f) const {
+  PayloadBase::dump(f);
+  f->dump_string("global_image_id", global_image_id);
+  f->dump_string("peer_mirror_uuid", peer_mirror_uuid);
+}
+
 void SyncPayloadBase::encode(bufferlist &bl) const {
   PayloadBase::encode(bl);
   ::encode(sync_id, bl);
@@ -132,13 +150,13 @@ void UnknownPayload::dump(Formatter *f) const {
 }
 
 void NotifyMessage::encode(bufferlist& bl) const {
-  ENCODE_START(1, 1, bl);
+  ENCODE_START(2, 2, bl);
   boost::apply_visitor(EncodePayloadVisitor(bl), payload);
   ENCODE_FINISH(bl);
 }
 
 void NotifyMessage::decode(bufferlist::iterator& iter) {
-  DECODE_START(1, iter);
+  DECODE_START(2, iter);
 
   uint32_t notify_op;
   ::decode(notify_op, iter);
@@ -151,6 +169,9 @@ void NotifyMessage::decode(bufferlist::iterator& iter) {
   case NOTIFY_OP_IMAGE_RELEASE:
     payload = ImageReleasePayload();
     break;
+  case NOTIFY_OP_PEER_IMAGE_REMOVED:
+    payload = PeerImageRemovedPayload();
+    break;
   case NOTIFY_OP_SYNC_REQUEST:
     payload = SyncRequestPayload();
     break;
@@ -178,6 +199,9 @@ void NotifyMessage::generate_test_instances(std::list<NotifyMessage *> &o) {
   o.push_back(new NotifyMessage(ImageReleasePayload(1, "gid", "uuid", "id",
                                                     true)));
 
+  o.push_back(new NotifyMessage(PeerImageRemovedPayload()));
+  o.push_back(new NotifyMessage(PeerImageRemovedPayload(1, "gid", "uuid")));
+
   o.push_back(new NotifyMessage(SyncRequestPayload()));
   o.push_back(new NotifyMessage(SyncRequestPayload(1, "sync_id")));
 
@@ -193,6 +217,9 @@ std::ostream &operator<<(std::ostream &out, const NotifyOp &op) {
   case NOTIFY_OP_IMAGE_RELEASE:
     out << "ImageRelease";
     break;
+  case NOTIFY_OP_PEER_IMAGE_REMOVED:
+    out << "PeerImageRemoved";
+    break;
   case NOTIFY_OP_SYNC_REQUEST:
     out << "SyncRequest";
     break;

diff --git a/src/tools/rbd_mirror/instance_watcher/Types.h b/src/tools/rbd_mirror/instance_watcher/Types.h
index 53b333c34afc98da330d99b9e74d50f04cb3d2d1..d330f9b4ae78009b74c99daabf213735cafacf1d 100644 (file)
--- a/src/tools/rbd_mirror/instance_watcher/Types.h
+++ b/src/tools/rbd_mirror/instance_watcher/Types.h
@@ -19,10 +19,11 @@ namespace mirror {
 namespace instance_watcher {
 
 enum NotifyOp {
-  NOTIFY_OP_IMAGE_ACQUIRE  = 0,
-  NOTIFY_OP_IMAGE_RELEASE  = 1,
-  NOTIFY_OP_SYNC_REQUEST   = 2,
-  NOTIFY_OP_SYNC_START     = 3,
+  NOTIFY_OP_IMAGE_ACQUIRE      = 0,
+  NOTIFY_OP_IMAGE_RELEASE      = 1,
+  NOTIFY_OP_PEER_IMAGE_REMOVED = 2,
+  NOTIFY_OP_SYNC_REQUEST       = 3,
+  NOTIFY_OP_SYNC_START         = 4
 };
 
 struct PayloadBase {
@@ -94,6 +95,26 @@ struct ImageReleasePayload : public ImagePayloadBase {
   void dump(Formatter *f) const;
 };
 
+struct PeerImageRemovedPayload : public PayloadBase {
+  static const NotifyOp NOTIFY_OP = NOTIFY_OP_PEER_IMAGE_REMOVED;
+
+  std::string global_image_id;
+  std::string peer_mirror_uuid;
+
+  PeerImageRemovedPayload() {
+  }
+  PeerImageRemovedPayload(uint64_t request_id,
+                          const std::string& global_image_id,
+                          const std::string& peer_mirror_uuid)
+    : PayloadBase(request_id),
+      global_image_id(global_image_id), peer_mirror_uuid(peer_mirror_uuid) {
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(__u8 version, bufferlist::iterator &iter);
+  void dump(Formatter *f) const;
+};
+
 struct SyncPayloadBase : public PayloadBase {
   std::string sync_id;
 
@@ -144,6 +165,7 @@ struct UnknownPayload {
 
 typedef boost::variant<ImageAcquirePayload,
                        ImageReleasePayload,
+                       PeerImageRemovedPayload,
                        SyncRequestPayload,
                        SyncStartPayload,
                        UnknownPayload> Payload;