From 1afc6f50506d6536df8921b3fdac66bb2877db1e Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 14 Dec 2017 12:33:05 -0500
Subject: [PATCH] rbd-mirror: new 'rbd_mirroring_delete_delay' configuration
 option

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/common/options.cc                         |  4 +
 src/librbd/ImageCtx.cc                        |  2 +
 src/librbd/ImageCtx.h                         |  1 +
 src/test/librbd/mock/MockImageCtx.h           |  2 +
 .../test_mock_TrashMoveRequest.cc             | 78 +++++++++++++++++--
 .../image_deleter/TrashMoveRequest.cc         |  3 +-
 6 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index d5fc04a3b39..00342993ddf 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5873,6 +5873,10 @@ static std::vector<Option> get_rbd_options() {
     .set_default(false)
     .set_description("automatically start image resync after mirroring is disconnected due to being laggy"),
 
+    Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("time-delay in seconds for rbd-mirror delete propagation"),
+
     Option("rbd_mirroring_replay_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description("time-delay in seconds for rbd-mirror asynchronous replication"),
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 9925b80cbc6..90abc4f5303 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -1020,6 +1020,7 @@ struct C_InvalidateCache : public Context {
         "rbd_journal_max_payload_bytes", false)(
         "rbd_journal_max_concurrent_object_sets", false)(
         "rbd_mirroring_resync_after_disconnect", false)(
+        "rbd_mirroring_delete_delay", false)(
         "rbd_mirroring_replay_delay", false)(
         "rbd_skip_partial_discard", false)(
 	"rbd_qos_iops_limit", false);
@@ -1080,6 +1081,7 @@ struct C_InvalidateCache : public Context {
     ASSIGN_OPTION(journal_max_payload_bytes, uint64_t);
     ASSIGN_OPTION(journal_max_concurrent_object_sets, int64_t);
     ASSIGN_OPTION(mirroring_resync_after_disconnect, bool);
+    ASSIGN_OPTION(mirroring_delete_delay, uint64_t);
     ASSIGN_OPTION(mirroring_replay_delay, int64_t);
     ASSIGN_OPTION(skip_partial_discard, bool);
     ASSIGN_OPTION(blkin_trace_all, bool);
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 89ece998879..41635fd4cb5 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -196,6 +196,7 @@ namespace librbd {
     uint32_t journal_max_payload_bytes;
     int journal_max_concurrent_object_sets;
     bool mirroring_resync_after_disconnect;
+    uint64_t mirroring_delete_delay;
     int mirroring_replay_delay;
     bool skip_partial_discard;
     bool blkin_trace_all;
diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index 17697a12d19..1d074fc8893 100644
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -106,6 +106,7 @@ struct MockImageCtx {
           image_ctx.journal_max_concurrent_object_sets),
       mirroring_resync_after_disconnect(
           image_ctx.mirroring_resync_after_disconnect),
+      mirroring_delete_delay(image_ctx.mirroring_delete_delay),
       mirroring_replay_delay(image_ctx.mirroring_replay_delay),
       non_blocking_aio(image_ctx.non_blocking_aio),
       blkin_trace_all(image_ctx.blkin_trace_all),
@@ -314,6 +315,7 @@ struct MockImageCtx {
   uint32_t journal_max_payload_bytes;
   int journal_max_concurrent_object_sets;
   bool mirroring_resync_after_disconnect;
+  uint64_t mirroring_delete_delay;
   int mirroring_replay_delay;
   bool non_blocking_aio;
   bool blkin_trace_all;
diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc
index 43eb524bb79..4e9074b6566 100644
--- a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc
+++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc
@@ -127,10 +127,13 @@ struct MoveRequest<MockTestImageCtx> {
                              const cls::rbd::TrashImageSpec& trash_image_spec,
                              Context* on_finish) {
     assert(s_instance != nullptr);
+    s_instance->construct(image_id, trash_image_spec);
     s_instance->on_finish = on_finish;
     return s_instance;
   }
 
+  MOCK_METHOD2(construct, void(const std::string&,
+                               const cls::rbd::TrashImageSpec&));
   MOCK_METHOD0(send, void());
 
   MoveRequest() {
@@ -265,7 +268,22 @@ public:
   }
 
   void expect_trash_move(MockLibrbdTrashMoveRequest& mock_trash_move_request,
-                         int r) {
+                         const std::string& image_name,
+                         const std::string& image_id,
+                         const boost::optional<uint32_t>& delay, int r) {
+    EXPECT_CALL(mock_trash_move_request, construct(image_id, _))
+      .WillOnce(WithArg<1>(Invoke([image_name, delay](const cls::rbd::TrashImageSpec& spec) {
+                             ASSERT_EQ(cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING,
+                                       spec.source);
+                             ASSERT_EQ(image_name, spec.name);
+                             if (delay) {
+                               utime_t time{spec.deletion_time};
+                               time += *delay;
+                               ASSERT_TRUE(time == spec.deferment_end_time);
+                             } else {
+                               ASSERT_EQ(spec.deletion_time, spec.deferment_end_time);
+                             }
+                           })));
     EXPECT_CALL(mock_trash_move_request, send())
       .WillOnce(Invoke([this, &mock_trash_move_request, r]() {
                   m_threads->work_queue->queue(mock_trash_move_request.on_finish, r);
@@ -306,7 +324,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, Success) {
   expect_acquire_lock(mock_image_ctx, 0);
 
   MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
-  expect_trash_move(mock_librbd_trash_move_request, 0);
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    {}, 0);
   expect_mirror_image_remove(m_local_io_ctx, 0);
 
   expect_close(mock_image_ctx, 0);
@@ -408,7 +427,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerDNE) {
   expect_acquire_lock(mock_image_ctx, 0);
 
   MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
-  expect_trash_move(mock_librbd_trash_move_request, 0);
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    {}, 0);
   expect_mirror_image_remove(m_local_io_ctx, 0);
 
   expect_close(mock_image_ctx, 0);
@@ -570,7 +590,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, TrashMoveError) {
   expect_acquire_lock(mock_image_ctx, 0);
 
   MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
-  expect_trash_move(mock_librbd_trash_move_request, -EINVAL);
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    {}, -EINVAL);
 
   expect_close(mock_image_ctx, 0);
   expect_destroy(mock_image_ctx);
@@ -607,7 +628,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, RemoveMirrorImageError) {
   expect_acquire_lock(mock_image_ctx, 0);
 
   MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
-  expect_trash_move(mock_librbd_trash_move_request, 0);
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    {}, 0);
   expect_mirror_image_remove(m_local_io_ctx, -EINVAL);
 
   expect_close(mock_image_ctx, 0);
@@ -648,7 +670,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) {
   expect_acquire_lock(mock_image_ctx, 0);
 
   MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
-  expect_trash_move(mock_librbd_trash_move_request, 0);
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    {}, 0);
   expect_mirror_image_remove(m_local_io_ctx, 0);
 
   expect_close(mock_image_ctx, -EINVAL);
@@ -666,6 +689,49 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) {
   ASSERT_EQ(0, ctx.wait());
 }
 
+TEST_F(TestMockImageDeleterTrashMoveRequest, DelayedDelation) {
+  librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx);
+  librbd::MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.mirroring_delete_delay = 600;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+
+  InSequence seq;
+  expect_mirror_image_get_image_id("image id", 0);
+
+  MockJournal mock_journal;
+  expect_get_tag_owner(mock_journal, "image id",
+                       librbd::Journal<>::ORPHAN_MIRROR_UUID, 0);
+  expect_mirror_image_set("image id", "global image id",
+                          cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0);
+
+  MockJournalResetRequest mock_journal_reset_request;
+  expect_journal_reset(mock_journal_reset_request, 0);
+
+  expect_set_journal_policy(mock_image_ctx);
+  expect_open(mock_image_ctx, 0);
+  expect_block_requests(mock_image_ctx);
+  expect_acquire_lock(mock_image_ctx, 0);
+
+  MockLibrbdTrashMoveRequest mock_librbd_trash_move_request;
+  expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id",
+                    600, 0);
+
+  expect_mirror_image_remove(m_local_io_ctx, 0);
+  expect_close(mock_image_ctx, 0);
+  expect_destroy(mock_image_ctx);
+
+  MockTrashWatcher mock_trash_watcher;
+  expect_notify_image_added(mock_trash_watcher, "image id");
+
+  C_SaferCond ctx;
+  auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id",
+                                          true,
+                                          m_local_image_ctx->op_work_queue,
+                                          &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+}
+
 } // namespace image_deleter
 } // namespace mirror
 } // namespace rbd
diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc
index 309894f5a07..bf52853277a 100644
--- a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc
+++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc
@@ -253,9 +253,10 @@ template <typename I>
 void TrashMoveRequest<I>::trash_move() {
   dout(10) << dendl;
 
-  // TODO support configurable "deletion delay"
   utime_t delete_time{ceph_clock_now()};
   utime_t deferment_end_time{delete_time};
+  deferment_end_time += m_image_ctx->mirroring_delete_delay;
+
   m_trash_image_spec = {
     cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, m_image_ctx->name, delete_time,
     deferment_end_time};
-- 
2.39.5