rbd-mirror: make pool watcher also refresh groups

Signed-off-by: Mykola Golub <mgolub@suse.com>
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

diff --git a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc
index bf4e9493f8ad48f33997a3af90bf32000e3370b8..05c9646381aec8bd4a5c8af0f577f533fafba667 100644 (file)
--- a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc
+++ b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshEntitiesRequest.cc
@@ -55,11 +55,45 @@ public:
                       Return(r)));
   }
 
+  void expect_mirror_group_list(
+      librados::IoCtx &io_ctx,
+      const std::map<std::string, cls::rbd::MirrorGroup> &groups, int r) {
+    bufferlist bl;
+    encode(groups, bl);
+
+    EXPECT_CALL(get_mock_io_ctx(io_ctx),
+                exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_group_list"),
+                     _, _, _, _))
+      .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) {
+                                          *out_bl = bl;
+                                        })),
+                      Return(r)));
+  }
+
+  void expect_group_image_list(
+      librados::IoCtx &io_ctx, const std::string &group_id,
+      const std::vector<cls::rbd::GroupImageStatus> &images, int r) {
+    bufferlist bl;
+    encode(images, bl);
+
+    EXPECT_CALL(get_mock_io_ctx(io_ctx),
+                exec(librbd::util::group_header_name(group_id), _, StrEq("rbd"),
+                     StrEq("group_image_list"), _, _, _, _))
+      .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) {
+                                          *out_bl = bl;
+                                        })),
+                      Return(r)));
+  }
 };
 
 TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, Success) {
   InSequence seq;
   expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0);
+  cls::rbd::MirrorGroup mirror_group =
+    {"global id", cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT,
+     cls::rbd::MIRROR_GROUP_STATE_ENABLED};
+  expect_mirror_group_list(m_remote_io_ctx, {{"local id", mirror_group}}, 0);
+  expect_group_image_list(m_remote_io_ctx, "local id", {{}, {}, {}}, 0);
 
   C_SaferCond ctx;
   std::map<MirrorEntity, std::string> entities;
@@ -70,25 +104,50 @@ TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, Success) {
   ASSERT_EQ(0, ctx.wait());
 
   std::map<MirrorEntity, std::string> expected_entities =
-    {{{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"}};
+    {{{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"},
+     {{MIRROR_ENTITY_TYPE_GROUP, "global id", 3}, "local id"}};
   ASSERT_EQ(expected_entities, entities);
 }
 
 TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, LargeDirectory) {
   InSequence seq;
-  std::map<std::string, std::string> mirror_list;
+  std::map<std::string, std::string> mirror_image_list;
   std::map<MirrorEntity, std::string> expected_entities;
   for (uint32_t idx = 1; idx <= 1024; ++idx) {
-    mirror_list.insert(std::make_pair("local id " + stringify(idx),
-                                      "global id " + stringify(idx)));
+    mirror_image_list.insert({"local id " + stringify(idx),
+                              "global id " + stringify(idx)});
     expected_entities.insert(
       {{MIRROR_ENTITY_TYPE_IMAGE, "global id " + stringify(idx), 1},
        "local id " + stringify(idx)});
   }
 
-  expect_mirror_image_list(m_remote_io_ctx, mirror_list, 0);
+  expect_mirror_image_list(m_remote_io_ctx, mirror_image_list, 0);
   expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0);
 
+  std::map<std::string, cls::rbd::MirrorGroup> mirror_group_list;
+  for (uint32_t idx = 1; idx <= 1024; ++idx) {
+    cls::rbd::MirrorGroup mirror_group =
+      {"global id " + stringify(idx),
+       cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT,
+       cls::rbd::MIRROR_GROUP_STATE_ENABLED};
+
+    mirror_group_list.insert({"local id " + stringify(idx), mirror_group});
+    expected_entities.insert(
+      {{MIRROR_ENTITY_TYPE_GROUP, "global id " + stringify(idx), 2},
+       "local id " + stringify(idx)});
+  }
+
+  expect_mirror_group_list(m_remote_io_ctx, mirror_group_list, 0);
+  cls::rbd::MirrorGroup mirror_group =
+    {"global id", cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT,
+     cls::rbd::MIRROR_GROUP_STATE_ENABLED};
+  expect_mirror_group_list(m_remote_io_ctx, {{"local id", mirror_group}}, 0);
+
+  expect_group_image_list(m_remote_io_ctx, "local id", {{}, {}}, 0);
+  for (auto &[group_id, _] : mirror_group_list) {
+    expect_group_image_list(m_remote_io_ctx, group_id, {{}, {}}, 0);
+  }
+
   C_SaferCond ctx;
   std::map<MirrorEntity, std::string> entities;
   MockRefreshEntitiesRequest *req = new MockRefreshEntitiesRequest(
@@ -99,6 +158,8 @@ TEST_F(TestMockPoolWatcherRefreshEntitiesRequest, LargeDirectory) {
 
   expected_entities.insert(
     {{MIRROR_ENTITY_TYPE_IMAGE, "global id", 1}, "local id"});
+  expected_entities.insert(
+    {{MIRROR_ENTITY_TYPE_GROUP, "global id", 2}, "local id"});
   ASSERT_EQ(expected_entities, entities);
 }
 

diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc
index 25d84d4bfb3f73efbe7133b96e9a5c171f0df2ea..78f0966cf65574d754bb99845e6b1295756f5cac 100644 (file)
--- a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc
+++ b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.cc
@@ -71,7 +71,111 @@ void RefreshEntitiesRequest<I>::handle_mirror_image_list(int r) {
     return;
   }
 
-  finish(0);
+  m_start_after = {};
+  mirror_group_list();
+}
+
+template <typename I>
+void RefreshEntitiesRequest<I>::mirror_group_list() {
+  dout(10) << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::mirror_group_list_start(&op, m_start_after, MAX_RETURN);
+
+  m_out_bl.clear();
+  librados::AioCompletion *aio_comp = create_rados_callback<
+    RefreshEntitiesRequest<I>,
+    &RefreshEntitiesRequest<I>::handle_mirror_group_list>(this);
+  int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void RefreshEntitiesRequest<I>::handle_mirror_group_list(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  std::map<std::string, cls::rbd::MirrorGroup> groups;
+  if (r == 0) {
+    auto it = m_out_bl.cbegin();
+    r = librbd::cls_client::mirror_group_list_finish(&it, &groups);
+  }
+
+  if (r < 0 && r != -ENOENT) {
+    derr << "failed to list mirrored groups: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  m_groups.insert(groups.begin(), groups.end());
+
+  if (groups.size() == MAX_RETURN) {
+    m_start_after = groups.rbegin()->first;
+    mirror_group_list();
+    return;
+  }
+
+  // XXXMG: should we just provide group_size field in cls::rbd::MirrorGroup
+  // instead of listing group images just to learn their count?
+  group_image_list();
+}
+
+template <typename I>
+void RefreshEntitiesRequest<I>::group_image_list() {
+  if (m_groups.empty()) {
+    finish(0);
+    return;
+  }
+
+  auto &group_id = m_groups.begin()->first;
+
+  dout(10) << group_id << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::group_image_list_start(
+      &op, m_start_group_image_list_after, MAX_RETURN);
+  m_out_bl.clear();
+  librados::AioCompletion *aio_comp = create_rados_callback<
+    RefreshEntitiesRequest<I>,
+    &RefreshEntitiesRequest<I>::handle_group_image_list>(this);
+  int r = m_remote_io_ctx.aio_operate(librbd::util::group_header_name(group_id),
+                                      aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void RefreshEntitiesRequest<I>::handle_group_image_list(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  std::vector<cls::rbd::GroupImageStatus> images;
+  if (r == 0) {
+    auto iter = m_out_bl.cbegin();
+    r = librbd::cls_client::group_image_list_finish(&iter, &images);
+  }
+
+  if (r < 0) {
+    derr << "error listing remote group: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  auto image_count = images.size();
+  m_group_size += image_count;
+  if (image_count == MAX_RETURN) {
+    m_start_group_image_list_after = images.rbegin()->spec;
+  } else {
+    auto group_it = m_groups.begin();
+    auto &global_group_id = group_it->second.global_group_id;
+    m_entities->insert(
+      {{MIRROR_ENTITY_TYPE_GROUP, global_group_id, m_group_size},
+       group_it->first});
+    m_groups.erase(group_it);
+    m_start_group_image_list_after = {};
+    m_group_size = 0;
+  }
+
+  group_image_list();
 }
 
 template <typename I>

diff --git a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h
index ab1957a82b0b9a9c9507ec19c524256d136ee957..70a5bda5ec53cfccabab8f21e0b145de1e0621e6 100644 (file)
--- a/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h
+++ b/src/tools/rbd_mirror/pool_watcher/RefreshEntitiesRequest.h
@@ -6,6 +6,7 @@
 
 #include "include/buffer.h"
 #include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
 #include "tools/rbd_mirror/Types.h"
 #include <string>
 
@@ -48,6 +49,16 @@ private:
    *    v   v             | (more images)
    * MIRROR_IMAGE_LIST ---/
    *    |
+   *    |   /-------------\
+   *    |   |             |
+   *    v   v             | (more groups)
+   * MIRROR_GROUP_LIST ---/
+   *    |
+   *    |   /-------------\
+   *    |   |             |
+   *    v   v             | (for every group)
+   * GROUP_IMAGE_LIST ----/
+   *    |
    *    v
    * <finish>
    *
@@ -61,9 +72,19 @@ private:
   bufferlist m_out_bl;
   std::string m_start_after;
 
+  std::map<std::string, cls::rbd::MirrorGroup> m_groups;
+  cls::rbd::GroupImageSpec m_start_group_image_list_after;
+  size_t m_group_size = 0;
+
   void mirror_image_list();
   void handle_mirror_image_list(int r);
 
+  void mirror_group_list();
+  void handle_mirror_group_list(int r);
+
+  void group_image_list();
+  void handle_group_image_list(int r);
+
   void finish(int r);
 
 };