cls/rbd: methods to track rbd-mirror instances

Signed-off-by: Mykola Golub <mgolub@mirantis.com>

diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc
index ae8cf9185d163f510771e331caab9ba3596b7a08..fd4e35d6489b37db346152f0fd6bf2e9a1d59fac 100644 (file)
--- a/src/cls/rbd/cls_rbd.cc
+++ b/src/cls/rbd/cls_rbd.cc
@@ -3096,6 +3096,7 @@ static const std::string PEER_KEY_PREFIX("mirror_peer_");
 static const std::string IMAGE_KEY_PREFIX("image_");
 static const std::string GLOBAL_KEY_PREFIX("global_");
 static const std::string STATUS_GLOBAL_KEY_PREFIX("status_global_");
+static const std::string INSTANCE_KEY_PREFIX("instance_");
 
 std::string peer_key(const std::string &uuid) {
   return PEER_KEY_PREFIX + uuid;
@@ -3113,6 +3114,10 @@ std::string status_global_key(const string &global_id) {
   return STATUS_GLOBAL_KEY_PREFIX + global_id;
 }
 
+std::string instance_key(const string &instance_id) {
+  return INSTANCE_KEY_PREFIX + instance_id;
+}
+
 int uuid_get(cls_method_context_t hctx, std::string *mirror_uuid) {
   bufferlist mirror_uuid_bl;
   int r = cls_cxx_map_get_val(hctx, mirror::UUID, &mirror_uuid_bl);
@@ -3599,6 +3604,56 @@ int image_status_remove_down(cls_method_context_t hctx) {
   return 0;
 }
 
+int instances_list(cls_method_context_t hctx,
+                   std::vector<std::string> *instance_ids) {
+  std::string last_read = INSTANCE_KEY_PREFIX;
+  int max_read = RBD_MAX_KEYS_READ;
+  int r = max_read;
+  while (r == max_read) {
+    std::map<std::string, bufferlist> vals;
+    r = cls_cxx_map_get_vals(hctx, last_read, INSTANCE_KEY_PREFIX.c_str(),
+                             max_read, &vals);
+    if (r < 0) {
+      if (r != -ENOENT) {
+       CLS_ERR("error reading mirror instances: %s", cpp_strerror(r).c_str());
+      }
+      return r;
+    }
+
+    for (auto &it : vals) {
+      instance_ids->push_back(it.first.substr(INSTANCE_KEY_PREFIX.size()));
+    }
+
+    if (!vals.empty()) {
+      last_read = vals.rbegin()->first;
+    }
+  }
+  return 0;
+}
+
+int instances_add(cls_method_context_t hctx, const string &instance_id) {
+  bufferlist bl;
+
+  int r = cls_cxx_map_set_val(hctx, instance_key(instance_id), &bl);
+  if (r < 0) {
+    CLS_ERR("error setting mirror instance %s: %s", instance_id.c_str(),
+            cpp_strerror(r).c_str());
+    return r;
+  }
+  return 0;
+}
+
+int instances_remove(cls_method_context_t hctx, const string &instance_id) {
+
+  int r = cls_cxx_map_remove_key(hctx, instance_key(instance_id));
+  if (r < 0) {
+    CLS_ERR("error removing mirror instance %s: %s", instance_id.c_str(),
+            cpp_strerror(r).c_str());
+    return r;
+  }
+  return 0;
+}
+
 } // namespace mirror
 
 /**
@@ -4268,6 +4323,75 @@ int mirror_image_status_remove_down(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+/**
+ * Input:
+ * none
+ *
+ * Output:
+ * @param std::vector<std::string>: instance ids
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_instances_list(cls_method_context_t hctx, bufferlist *in,
+                          bufferlist *out) {
+  std::vector<std::string> instance_ids;
+
+  int r = mirror::instances_list(hctx, &instance_ids);
+  if (r < 0) {
+    return r;
+  }
+
+  ::encode(instance_ids, *out);
+  return 0;
+}
+
+/**
+ * Input:
+ * @param instance_id (std::string)
+ *
+ * Output:
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_instances_add(cls_method_context_t hctx, bufferlist *in,
+                         bufferlist *out) {
+  std::string instance_id;
+  try {
+    bufferlist::iterator iter = in->begin();
+    ::decode(instance_id, iter);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  int r = mirror::instances_add(hctx, instance_id);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+/**
+ * Input:
+ * @param instance_id (std::string)
+ *
+ * Output:
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_instances_remove(cls_method_context_t hctx, bufferlist *in,
+                            bufferlist *out) {
+  std::string instance_id;
+  try {
+    bufferlist::iterator iter = in->begin();
+    ::decode(instance_id, iter);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  int r = mirror::instances_remove(hctx, instance_id);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
 /**
  * Initialize the header with basic metadata.
  * Everything is stored as key/value pairs as omaps in the header object.
@@ -4825,6 +4949,9 @@ CLS_INIT(rbd)
   cls_method_handle_t h_mirror_image_status_list;
   cls_method_handle_t h_mirror_image_status_get_summary;
   cls_method_handle_t h_mirror_image_status_remove_down;
+  cls_method_handle_t h_mirror_instances_list;
+  cls_method_handle_t h_mirror_instances_add;
+  cls_method_handle_t h_mirror_instances_remove;
   cls_method_handle_t h_group_create;
   cls_method_handle_t h_group_dir_list;
   cls_method_handle_t h_group_dir_add;
@@ -5060,6 +5187,15 @@ CLS_INIT(rbd)
                           CLS_METHOD_RD | CLS_METHOD_WR,
                           mirror_image_status_remove_down,
                          &h_mirror_image_status_remove_down);
+  cls_register_cxx_method(h_class, "mirror_instances_list", CLS_METHOD_RD,
+                          mirror_instances_list, &h_mirror_instances_list);
+  cls_register_cxx_method(h_class, "mirror_instances_add",
+                          CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PROMOTE,
+                          mirror_instances_add, &h_mirror_instances_add);
+  cls_register_cxx_method(h_class, "mirror_instances_remove",
+                          CLS_METHOD_RD | CLS_METHOD_WR,
+                          mirror_instances_remove,
+                          &h_mirror_instances_remove);
   /* methods for the consistency groups feature */
   cls_register_cxx_method(h_class, "group_create",
                          CLS_METHOD_RD | CLS_METHOD_WR,

diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc
index 804f28386a91f30915aa88af0422b4039dc31ed7..fda79678366d5abef767c192fcac4e68d7166a86 100644 (file)
--- a/src/cls/rbd/cls_rbd_client.cc
+++ b/src/cls/rbd/cls_rbd_client.cc
@@ -1738,6 +1738,69 @@ namespace librbd {
       op->exec("rbd", "mirror_image_status_remove_down", bl);
     }
 
+    void mirror_instances_list_start(librados::ObjectReadOperation *op) {
+      bufferlist bl;
+      op->exec("rbd", "mirror_instances_list", bl);
+    }
+
+    int mirror_instances_list_finish(bufferlist::iterator *iter,
+                                     std::vector<std::string> *instance_ids) {
+      instance_ids->clear();
+      try {
+       ::decode(*instance_ids, *iter);
+      } catch (const buffer::error &err) {
+       return -EBADMSG;
+      }
+      return 0;
+    }
+
+    int mirror_instances_list(librados::IoCtx *ioctx,
+                              std::vector<std::string> *instance_ids) {
+      librados::ObjectReadOperation op;
+      mirror_instances_list_start(&op);
+
+      bufferlist out_bl;
+      int r = ioctx->operate(RBD_MIRROR_LEADER, &op, &out_bl);
+      if (r < 0) {
+       return r;
+      }
+
+      bufferlist::iterator iter = out_bl.begin();
+      r = mirror_instances_list_finish(&iter, instance_ids);
+      if (r < 0) {
+       return r;
+      }
+      return 0;
+    }
+
+    void mirror_instances_add(librados::ObjectWriteOperation *op,
+                              const std::string &instance_id) {
+      bufferlist bl;
+      ::encode(instance_id, bl);
+      op->exec("rbd", "mirror_instances_add", bl);
+    }
+
+    int mirror_instances_add(librados::IoCtx *ioctx,
+                             const std::string &instance_id) {
+      librados::ObjectWriteOperation op;
+      mirror_instances_add(&op, instance_id);
+      return ioctx->operate(RBD_MIRROR_LEADER, &op);
+    }
+
+    void mirror_instances_remove(librados::ObjectWriteOperation *op,
+                                 const std::string &instance_id) {
+      bufferlist bl;
+      ::encode(instance_id, bl);
+      op->exec("rbd", "mirror_instances_remove", bl);
+    }
+
+    int mirror_instances_remove(librados::IoCtx *ioctx,
+                                const std::string &instance_id) {
+      librados::ObjectWriteOperation op;
+      mirror_instances_remove(&op, instance_id);
+      return ioctx->operate(RBD_MIRROR_LEADER, &op);
+    }
+
     // Consistency groups functions
     int group_create(librados::IoCtx *ioctx, const std::string &oid)
     {

diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h
index d73d70de07b3e1c34ef49d07b6d0cfbca8ba87a3..256dcb5f595044c77e3bcce0aef85d0b838bdf4f 100644 (file)
--- a/src/cls/rbd/cls_rbd_client.h
+++ b/src/cls/rbd/cls_rbd_client.h
@@ -354,6 +354,20 @@ namespace librbd {
     int mirror_image_status_remove_down(librados::IoCtx *ioctx);
     void mirror_image_status_remove_down(librados::ObjectWriteOperation *op);
 
+    void mirror_instances_list_start(librados::ObjectReadOperation *op);
+    int mirror_instances_list_finish(bufferlist::iterator *iter,
+                                     std::vector<std::string> *instance_ids);
+    int mirror_instances_list(librados::IoCtx *ioctx,
+                              std::vector<std::string> *instance_ids);
+    void mirror_instances_add(librados::ObjectWriteOperation *op,
+                              const std::string &instance_id);
+    int mirror_instances_add(librados::IoCtx *ioctx,
+                             const std::string &instance_id);
+    void mirror_instances_remove(librados::ObjectWriteOperation *op,
+                                 const std::string &instance_id);
+    int mirror_instances_remove(librados::IoCtx *ioctx,
+                                const std::string &instance_id);
+
     // Consistency groups functions
     int group_create(librados::IoCtx *ioctx, const std::string &oid);
     int group_dir_list(librados::IoCtx *ioctx, const std::string &oid,

diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc
index c9745f3380defd26c8779ae6b7c59244c461bdfa..97fc597a9a5032a97d9e1460d43f6c9f405a1623 100644 (file)
--- a/src/test/cls_rbd/test_cls_rbd.cc
+++ b/src/test/cls_rbd/test_cls_rbd.cc
@@ -1905,6 +1905,38 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_EQ(0U, statuses.size());
 }
 
+TEST_F(TestClsRbd, mirror_instances) {
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), ioctx));
+  ioctx.remove(RBD_MIRROR_LEADER);
+
+  std::vector<std::string> instance_ids;
+  ASSERT_EQ(-ENOENT, mirror_instances_list(&ioctx, &instance_ids));
+
+  ASSERT_EQ(0, ioctx.create(RBD_MIRROR_LEADER, true));
+  ASSERT_EQ(0, mirror_instances_list(&ioctx, &instance_ids));
+  ASSERT_EQ(0U, instance_ids.size());
+
+  ASSERT_EQ(0, mirror_instances_add(&ioctx, "instance_id1"));
+  ASSERT_EQ(0, mirror_instances_list(&ioctx, &instance_ids));
+  ASSERT_EQ(1U, instance_ids.size());
+  ASSERT_EQ(instance_ids[0], "instance_id1");
+
+  ASSERT_EQ(0, mirror_instances_add(&ioctx, "instance_id1"));
+  ASSERT_EQ(0, mirror_instances_add(&ioctx, "instance_id2"));
+  ASSERT_EQ(0, mirror_instances_list(&ioctx, &instance_ids));
+  ASSERT_EQ(2U, instance_ids.size());
+
+  ASSERT_EQ(0, mirror_instances_remove(&ioctx, "instance_id1"));
+  ASSERT_EQ(0, mirror_instances_list(&ioctx, &instance_ids));
+  ASSERT_EQ(1U, instance_ids.size());
+  ASSERT_EQ(instance_ids[0], "instance_id2");
+
+  ASSERT_EQ(0, mirror_instances_remove(&ioctx, "instance_id2"));
+  ASSERT_EQ(0, mirror_instances_list(&ioctx, &instance_ids));
+  ASSERT_EQ(0U, instance_ids.size());
+}
+
 TEST_F(TestClsRbd, group_create) {
   librados::IoCtx ioctx;
   ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), ioctx));