]> git-server-git.apps.pok.os.sepia.ceph.com Git - ceph.git/commitdiff
cls/rbd: store remote mirror image site status on disk
authorJason Dillaman <dillaman@redhat.com>
Fri, 20 Sep 2019 01:41:51 +0000 (21:41 -0400)
committerJason Dillaman <dillaman@redhat.com>
Tue, 8 Oct 2019 15:16:46 +0000 (11:16 -0400)
The image status get/list/summary methods now include the status
for peers that we can transmit images towards (i.e. remotes can
receive images).

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
src/cls/rbd/cls_rbd.cc
src/cls/rbd/cls_rbd_client.cc
src/cls/rbd/cls_rbd_client.h
src/librbd/api/Mirror.cc
src/test/cls_rbd/test_cls_rbd.cc

index eb29f5c7e18caa889da0ff17e36865d24a993239..9af11cd8539c3ff8fcf550980811f2f9bd19ddc3 100644 (file)
@@ -4610,6 +4610,7 @@ static const std::string PEER_KEY_PREFIX("mirror_peer_");
 static const std::string IMAGE_KEY_PREFIX("image_");
 static const std::string GLOBAL_KEY_PREFIX("global_");
 static const std::string STATUS_GLOBAL_KEY_PREFIX("status_global_");
+static const std::string REMOTE_STATUS_GLOBAL_KEY_PREFIX("remote_status_global_");
 static const std::string INSTANCE_KEY_PREFIX("instance_");
 static const std::string MIRROR_IMAGE_MAP_KEY_PREFIX("image_map_");
 
@@ -4625,8 +4626,18 @@ std::string global_key(const string &global_id) {
   return GLOBAL_KEY_PREFIX + global_id;
 }
 
-std::string status_global_key(const string &global_id) {
-  return STATUS_GLOBAL_KEY_PREFIX + global_id;
+std::string remote_status_global_key(const std::string& global_id,
+                                     const std::string& fsid) {
+  return REMOTE_STATUS_GLOBAL_KEY_PREFIX + global_id + "_" + fsid;
+}
+
+std::string status_global_key(const std::string& global_id,
+                              const std::string& fsid) {
+  if (fsid == cls::rbd::MirrorImageSiteStatus::LOCAL_FSID) {
+    return STATUS_GLOBAL_KEY_PREFIX + global_id;
+  } else {
+    return remote_status_global_key(global_id, fsid);
+  }
 }
 
 std::string instance_key(const string &instance_id) {
@@ -4995,6 +5006,9 @@ int image_set(cls_method_context_t hctx, const string &image_id,
   return 0;
 }
 
+int image_status_remove(cls_method_context_t hctx,
+                       const string &global_image_id);
+
 int image_remove(cls_method_context_t hctx, const string &image_id) {
   bufferlist bl;
   cls::rbd::MirrorImage mirror_image;
@@ -5025,11 +5039,8 @@ int image_remove(cls_method_context_t hctx, const string &image_id) {
     return r;
   }
 
-  r = cls_cxx_map_remove_key(hctx,
-                             status_global_key(mirror_image.global_image_id));
-  if (r < 0 && r != -ENOENT) {
-    CLS_ERR("error removing global status for image '%s': %s", image_id.c_str(),
-           cpp_strerror(r).c_str());
+  r = image_status_remove(hctx, mirror_image.global_image_id);
+  if (r < 0) {
     return r;
   }
 
@@ -5039,6 +5050,7 @@ int image_remove(cls_method_context_t hctx, const string &image_id) {
 int image_status_set(cls_method_context_t hctx, const string &global_image_id,
                     const cls::rbd::MirrorImageSiteStatus &status) {
   cls::rbd::MirrorImageSiteStatusOnDisk ondisk_status(status);
+  ondisk_status.fsid = ""; // fsid stored in key
   ondisk_status.up = false;
   ondisk_status.last_update = ceph_clock_now();
 
@@ -5049,37 +5061,85 @@ int image_status_set(cls_method_context_t hctx, const string &global_image_id,
   bufferlist bl;
   encode(ondisk_status, bl, cls_get_features(hctx));
 
-  r = cls_cxx_map_set_val(hctx, status_global_key(global_image_id), &bl);
+  r = cls_cxx_map_set_val(hctx, status_global_key(global_image_id,
+                                                  status.fsid), &bl);
   if (r < 0) {
-    CLS_ERR("error setting status for mirrored image, global id '%s': %s",
-           global_image_id.c_str(), cpp_strerror(r).c_str());
+    CLS_ERR("error setting status for mirrored image, global id '%s', "
+            "site '%s': %s", global_image_id.c_str(), status.fsid.c_str(),
+            cpp_strerror(r).c_str());
     return r;
   }
   return 0;
 }
 
+int get_remote_image_status_fsids(cls_method_context_t hctx,
+                                  const std::string& global_image_id,
+                                  std::set<std::string>* fsids) {
+  std::string filter = remote_status_global_key(global_image_id, "");
+  std::string last_read = filter;
+  int max_read = RBD_MAX_KEYS_READ;
+  bool more = true;
+
+  do {
+    std::set<std::string> keys;
+    int r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys, &more);
+    if (r < 0) {
+      return r;
+    }
+
+    for (auto& key : keys) {
+      if (key.find(filter) != 0 && key.length() <= filter.length()) {
+        break;
+      }
+
+      fsids->insert(key.substr(filter.length()));
+    }
+
+    if (!keys.empty()) {
+      last_read = *keys.rbegin();
+    }
+  } while (more);
+
+  return 0;
+}
+
 int image_status_remove(cls_method_context_t hctx,
                        const string &global_image_id) {
-
-  int r = cls_cxx_map_remove_key(hctx, status_global_key(global_image_id));
-  if (r < 0) {
-    CLS_ERR("error removing status for mirrored image, global id '%s': %s",
-           global_image_id.c_str(), cpp_strerror(r).c_str());
+  // remove all local/remote image statuses
+  std::set<std::string> fsids;
+  int r = get_remote_image_status_fsids(hctx, global_image_id, &fsids);
+  if (r < 0 && r != -ENOENT) {
     return r;
   }
+
+  fsids.insert(cls::rbd::MirrorImageSiteStatus::LOCAL_FSID);
+  for (auto& fsid : fsids) {
+    CLS_LOG(20, "removing status object for fsid %s", fsid.c_str());
+    auto key = status_global_key(global_image_id, fsid);
+    r = cls_cxx_map_remove_key(hctx, key);
+    if (r < 0 && r != -ENOENT) {
+      CLS_ERR("error removing stale status for key '%s': %s",
+              key.c_str(), cpp_strerror(r).c_str());
+      return r;
+    }
+  }
+
   return 0;
 }
 
 int image_status_get(cls_method_context_t hctx, const string &global_image_id,
+                     const std::string& fsid,
                      const std::set<entity_inst_t> &watchers,
-                    cls::rbd::MirrorImageStatus *status) {
-
+                     cls::rbd::MirrorImageSiteStatus *status) {
   bufferlist bl;
-  int r = cls_cxx_map_get_val(hctx, status_global_key(global_image_id), &bl);
+  int r = cls_cxx_map_get_val(hctx, status_global_key(global_image_id,
+                                                      fsid), &bl);
   if (r < 0) {
     if (r != -ENOENT) {
-      CLS_ERR("error reading status for mirrored image, global id '%s': '%s'",
-             global_image_id.c_str(), cpp_strerror(r).c_str());
+      CLS_ERR("error reading status for mirrored image, global id '%s', "
+              "site '%s': '%s'",
+             global_image_id.c_str(), fsid.c_str(),
+              cpp_strerror(r).c_str());
     }
     return r;
   }
@@ -5094,9 +5154,55 @@ int image_status_get(cls_method_context_t hctx, const string &global_image_id,
     return -EIO;
   }
 
-  ondisk_status.fsid = cls::rbd::MirrorImageSiteStatus::LOCAL_FSID;
-  ondisk_status.up = (watchers.find(ondisk_status.origin) != watchers.end());
-  *status = {{ondisk_status}};
+  *status = static_cast<cls::rbd::MirrorImageSiteStatus>(ondisk_status);
+  status->up = (watchers.find(ondisk_status.origin) != watchers.end());
+  return 0;
+}
+
+int image_status_get(cls_method_context_t hctx, const string &global_image_id,
+                     const std::set<entity_inst_t> &watchers,
+                     cls::rbd::MirrorImageStatus *status) {
+  status->mirror_image_site_statuses.clear();
+
+  std::set<std::string> fsids;
+  int r = get_remote_image_status_fsids(hctx, global_image_id, &fsids);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+
+  // collect local site status
+  cls::rbd::MirrorImageSiteStatus local_status;
+  r = image_status_get(
+    hctx, global_image_id, cls::rbd::MirrorImageSiteStatus::LOCAL_FSID,
+    watchers, &local_status);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+
+  if (r >= 0) {
+    local_status.fsid = cls::rbd::MirrorImageSiteStatus::LOCAL_FSID;
+    status->mirror_image_site_statuses.push_back(local_status);
+  }
+
+  // collect remote site status (TX to peer)
+  for (auto& fsid : fsids) {
+    cls::rbd::MirrorImageSiteStatus remote_status;
+    r = image_status_get(
+      hctx, global_image_id, fsid, watchers, &remote_status);
+    if (r < 0 && r != -ENOENT) {
+      return r;
+    }
+
+    if (r >= 0) {
+      remote_status.fsid = fsid;
+      status->mirror_image_site_statuses.push_back(remote_status);
+    }
+  }
+
+  if (status->mirror_image_site_statuses.empty()) {
+    return -ENOENT;
+  }
+
   return 0;
 }
 
@@ -5159,8 +5265,67 @@ int image_status_list(cls_method_context_t hctx,
   return 0;
 }
 
+cls::rbd::MirrorImageStatusState compute_image_status_summary_state(
+    cls::rbd::MirrorPeerDirection mirror_peer_direction,
+    const std::set<std::string>& tx_peer_fsids,
+    const cls::rbd::MirrorImageStatus& status) {
+  std::optional<cls::rbd::MirrorImageStatusState> state = {};
+
+  cls::rbd::MirrorImageSiteStatus local_status;
+  status.get_local_mirror_image_site_status(&local_status);
+
+  uint64_t unmatched_tx_peers = 0;
+  switch (mirror_peer_direction) {
+  case cls::rbd::MIRROR_PEER_DIRECTION_RX:
+    // if we are RX-only, summary is based on our local status
+    if (local_status.up) {
+      state = local_status.state;
+    }
+    break;
+  case cls::rbd::MIRROR_PEER_DIRECTION_RX_TX:
+    // if we are RX/TX, combine all statuses
+    if (local_status.up) {
+      state = local_status.state;
+    }
+    [[fallthrough]];
+  case cls::rbd::MIRROR_PEER_DIRECTION_TX:
+    // if we are TX-only, summary is based on remote status
+    unmatched_tx_peers = tx_peer_fsids.size();
+    for (auto& remote_status : status.mirror_image_site_statuses) {
+      if (remote_status.fsid == cls::rbd::MirrorImageSiteStatus::LOCAL_FSID) {
+        continue;
+      }
+
+      if (unmatched_tx_peers > 0 &&
+          tx_peer_fsids.count(remote_status.fsid) > 0) {
+        --unmatched_tx_peers;
+      }
+
+      auto remote_state = (remote_status.up ?
+        remote_status.state : cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN);
+      if (remote_status.state == cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR) {
+        state = remote_status.state;
+      } else if (!state) {
+        state = remote_state;
+      } else if (*state != cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR) {
+        state = std::min(*state, remote_state);
+      }
+    }
+    break;
+  default:
+    break;
+  }
+
+  if (!state || unmatched_tx_peers > 0) {
+    state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN;
+  }
+  return *state;
+}
+
 int image_status_get_summary(
     cls_method_context_t hctx,
+    cls::rbd::MirrorPeerDirection mirror_peer_direction,
+    const std::set<std::string>& tx_peer_fsids,
     std::map<cls::rbd::MirrorImageStatusState, int> *states) {
   std::set<entity_inst_t> watchers;
   int r = list_watchers(hctx, &watchers);
@@ -5202,12 +5367,14 @@ int image_status_get_summary(
       }
 
       cls::rbd::MirrorImageStatus status;
-      image_status_get(hctx, mirror_image.global_image_id, watchers, &status);
+      r = image_status_get(hctx, mirror_image.global_image_id, watchers,
+                           &status);
+      if (r < 0 && r != -ENOENT) {
+        return r;
+      }
 
-      cls::rbd::MirrorImageSiteStatus local_status;
-      r = status.get_local_mirror_image_site_status(&local_status);
-      cls::rbd::MirrorImageStatusState state = (r >= 0 && local_status.up ?
-        local_status.state : cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN);
+      auto state = compute_image_status_summary_state(
+        mirror_peer_direction, tx_peer_fsids, status);
       (*states)[state]++;
     }
 
@@ -5226,52 +5393,54 @@ int image_status_remove_down(cls_method_context_t hctx) {
     return r;
   }
 
-  string last_read = STATUS_GLOBAL_KEY_PREFIX;
-  int max_read = RBD_MAX_KEYS_READ;
-  bool more = true;
-  while (more) {
-    map<string, bufferlist> vals;
-    r = cls_cxx_map_get_vals(hctx, last_read, STATUS_GLOBAL_KEY_PREFIX,
-                            max_read, &vals, &more);
-    if (r < 0) {
-      if (r != -ENOENT) {
-        CLS_ERR("error reading mirrored images: %s", cpp_strerror(r).c_str());
+  std::vector<std::string> prefixes = {
+    STATUS_GLOBAL_KEY_PREFIX, REMOTE_STATUS_GLOBAL_KEY_PREFIX};
+  for (auto& prefix : prefixes) {
+    std::string last_read = prefix;
+    int max_read = RBD_MAX_KEYS_READ;
+    bool more = true;
+    while (more) {
+      std::map<std::string, bufferlist> vals;
+      r = cls_cxx_map_get_vals(hctx, last_read, prefix, max_read, &vals, &more);
+      if (r < 0) {
+        if (r != -ENOENT) {
+          CLS_ERR("error reading mirrored images: %s", cpp_strerror(r).c_str());
+        }
+        return r;
       }
-      return r;
-    }
 
-    for (auto &list_it : vals) {
-      const string &key = list_it.first;
+      for (auto &list_it : vals) {
+        const std::string &key = list_it.first;
 
-      if (0 != key.compare(0, STATUS_GLOBAL_KEY_PREFIX.size(),
-                          STATUS_GLOBAL_KEY_PREFIX)) {
-       break;
-      }
+        if (0 != key.compare(0, prefix.size(), prefix)) {
+          break;
+        }
 
-      cls::rbd::MirrorImageSiteStatusOnDisk status;
-      try {
-       auto it = list_it.second.cbegin();
-       status.decode_meta(it);
-      } catch (const buffer::error &err) {
-       CLS_ERR("could not decode status metadata for mirrored image '%s'",
-               key.c_str());
-       return -EIO;
-      }
+        cls::rbd::MirrorImageSiteStatusOnDisk status;
+        try {
+          auto it = list_it.second.cbegin();
+          status.decode_meta(it);
+        } catch (const buffer::error &err) {
+          CLS_ERR("could not decode status metadata for mirrored image '%s'",
+                  key.c_str());
+          return -EIO;
+        }
 
-      if (watchers.find(status.origin) == watchers.end()) {
-       CLS_LOG(20, "removing stale status object for key %s",
-               key.c_str());
-       int r1 = cls_cxx_map_remove_key(hctx, key);
-       if (r1 < 0) {
-         CLS_ERR("error removing stale status for key '%s': %s",
-                 key.c_str(), cpp_strerror(r1).c_str());
-         return r1;
-       }
+        if (watchers.find(status.origin) == watchers.end()) {
+          CLS_LOG(20, "removing stale status object for key %s",
+                  key.c_str());
+          int r1 = cls_cxx_map_remove_key(hctx, key);
+          if (r1 < 0) {
+            CLS_ERR("error removing stale status for key '%s': %s",
+                    key.c_str(), cpp_strerror(r1).c_str());
+            return r1;
+          }
+        }
       }
-    }
 
-    if (!vals.empty()) {
-      last_read = vals.rbegin()->first;
+      if (!vals.empty()) {
+        last_read = vals.rbegin()->first;
+      }
     }
   }
 
@@ -5282,8 +5451,12 @@ int image_instance_get(cls_method_context_t hctx,
                        const string &global_image_id,
                        const std::set<entity_inst_t> &watchers,
                        entity_inst_t *instance) {
+  // instance details only available for local site
   bufferlist bl;
-  int r = cls_cxx_map_get_val(hctx, status_global_key(global_image_id), &bl);
+  int r = cls_cxx_map_get_val(
+    hctx, status_global_key(global_image_id,
+                            cls::rbd::MirrorImageSiteStatus::LOCAL_FSID),
+    &bl);
   if (r < 0) {
     if (r != -ENOENT) {
       CLS_ERR("error reading status for mirrored image, global id '%s': '%s'",
@@ -5811,6 +5984,43 @@ int mirror_peer_set_cluster(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+/**
+ * Input:
+ * @param uuid (std::string)
+ * @param direction (uint8_t)
+ *
+ * Output:
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_peer_set_direction(cls_method_context_t hctx, bufferlist *in,
+                              bufferlist *out) {
+  std::string uuid;
+  cls::rbd::MirrorPeerDirection mirror_peer_direction;
+  try {
+    auto it = in->cbegin();
+    decode(uuid, it);
+    uint8_t direction;
+    decode(direction, it);
+    mirror_peer_direction = static_cast<cls::rbd::MirrorPeerDirection>(
+      direction);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  cls::rbd::MirrorPeer peer;
+  int r = mirror::read_peer(hctx, uuid, &peer);
+  if (r < 0) {
+    return r;
+  }
+
+  peer.mirror_peer_direction = mirror_peer_direction;
+  r = mirror::write_peer(hctx, peer);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
 /**
  * Input:
  * @param start_after which name to begin listing after
@@ -6115,7 +6325,7 @@ int mirror_image_status_list(cls_method_context_t hctx, bufferlist *in,
 
 /**
  * Input:
- * none
+ * @param std::vector<cls::rbd::MirrorPeer> - optional peers (backwards compatibility)
  *
  * Output:
  * @param std::map<cls::rbd::MirrorImageStatusState, int>: states counts
@@ -6123,9 +6333,37 @@ int mirror_image_status_list(cls_method_context_t hctx, bufferlist *in,
  */
 int mirror_image_status_get_summary(cls_method_context_t hctx, bufferlist *in,
                                    bufferlist *out) {
-  std::map<cls::rbd::MirrorImageStatusState, int> states;
+  std::vector<cls::rbd::MirrorPeer> peers;
+  try {
+    auto iter = in->cbegin();
+    if (!iter.end()) {
+      decode(peers, iter);
+    }
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  auto mirror_peer_direction = cls::rbd::MIRROR_PEER_DIRECTION_RX;
+  if (!peers.empty()) {
+    mirror_peer_direction = peers.begin()->mirror_peer_direction;
+  }
+
+  std::set<std::string> tx_peer_fsids;
+  for (auto& peer : peers) {
+    if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
+      continue;
+    }
 
-  int r = mirror::image_status_get_summary(hctx, &states);
+    tx_peer_fsids.insert(peer.fsid);
+    if (mirror_peer_direction != cls::rbd::MIRROR_PEER_DIRECTION_RX_TX &&
+        mirror_peer_direction != peer.mirror_peer_direction) {
+      mirror_peer_direction = cls::rbd::MIRROR_PEER_DIRECTION_RX_TX;
+    }
+  }
+
+  std::map<cls::rbd::MirrorImageStatusState, int> states;
+  int r = mirror::image_status_get_summary(hctx, mirror_peer_direction,
+                                           tx_peer_fsids, &states);
   if (r < 0) {
     return r;
   }
@@ -7729,6 +7967,7 @@ CLS_INIT(rbd)
   cls_method_handle_t h_mirror_peer_remove;
   cls_method_handle_t h_mirror_peer_set_client;
   cls_method_handle_t h_mirror_peer_set_cluster;
+  cls_method_handle_t h_mirror_peer_set_direction;
   cls_method_handle_t h_mirror_image_list;
   cls_method_handle_t h_mirror_image_get_image_id;
   cls_method_handle_t h_mirror_image_get;
@@ -8033,6 +8272,10 @@ CLS_INIT(rbd)
   cls_register_cxx_method(h_class, "mirror_peer_set_cluster",
                           CLS_METHOD_RD | CLS_METHOD_WR,
                           mirror_peer_set_cluster, &h_mirror_peer_set_cluster);
+  cls_register_cxx_method(h_class, "mirror_peer_set_direction",
+                          CLS_METHOD_RD | CLS_METHOD_WR,
+                          mirror_peer_set_direction,
+                          &h_mirror_peer_set_direction);
   cls_register_cxx_method(h_class, "mirror_image_list", CLS_METHOD_RD,
                           mirror_image_list, &h_mirror_image_list);
   cls_register_cxx_method(h_class, "mirror_image_get_image_id", CLS_METHOD_RD,
index d9d6d86d12318f5d3249ecc2d8c37ca23fe3a7fc..678df7c03bbf707741c5fe865bf6aa6df14b5262 100644 (file)
@@ -1922,6 +1922,22 @@ int mirror_peer_set_cluster(librados::IoCtx *ioctx,
   return 0;
 }
 
+int mirror_peer_set_direction(
+    librados::IoCtx *ioctx, const std::string &uuid,
+    cls::rbd::MirrorPeerDirection mirror_peer_direction) {
+  bufferlist in_bl;
+  encode(uuid, in_bl);
+  encode(static_cast<uint8_t>(mirror_peer_direction), in_bl);
+
+  bufferlist out_bl;
+  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_peer_set_direction",
+                      in_bl, out_bl);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
 void mirror_image_list_start(librados::ObjectReadOperation *op,
                              const std::string &start, uint64_t max_return)
 {
@@ -2167,10 +2183,12 @@ int mirror_image_status_list_finish(bufferlist::const_iterator *iter,
   return 0;
 }
 
-int mirror_image_status_get_summary(librados::IoCtx *ioctx,
-                                    std::map<cls::rbd::MirrorImageStatusState, int> *states) {
+int mirror_image_status_get_summary(
+    librados::IoCtx *ioctx,
+    const std::vector<cls::rbd::MirrorPeer>& mirror_peer_sites,
+    std::map<cls::rbd::MirrorImageStatusState, int> *states) {
   librados::ObjectReadOperation op;
-  mirror_image_status_get_summary_start(&op);
+  mirror_image_status_get_summary_start(&op, mirror_peer_sites);
 
   bufferlist out_bl;
   int r = ioctx->operate(RBD_MIRRORING, &op, &out_bl);
@@ -2186,13 +2204,17 @@ int mirror_image_status_get_summary(librados::IoCtx *ioctx,
   return 0;
 }
 
-void mirror_image_status_get_summary_start(librados::ObjectReadOperation *op) {
+void mirror_image_status_get_summary_start(
+    librados::ObjectReadOperation *op,
+    const std::vector<cls::rbd::MirrorPeer>& mirror_peer_sites) {
   bufferlist bl;
+  encode(mirror_peer_sites, bl);
   op->exec("rbd", "mirror_image_status_get_summary", bl);
 }
 
-int mirror_image_status_get_summary_finish(bufferlist::const_iterator *iter,
-                                           std::map<cls::rbd::MirrorImageStatusState, int> *states) {
+int mirror_image_status_get_summary_finish(
+    bufferlist::const_iterator *iter,
+    std::map<cls::rbd::MirrorImageStatusState, int> *states) {
   try {
     decode(*states, *iter);
   } catch (const buffer::error &err) {
index 70d77284e380ccc1bb57a793d72722c799121b55..854592062606ad3a10aa124de7b159af03598eb2 100644 (file)
@@ -402,6 +402,9 @@ int mirror_peer_set_client(librados::IoCtx *ioctx,
 int mirror_peer_set_cluster(librados::IoCtx *ioctx,
                             const std::string &uuid,
                             const std::string &cluster_name);
+int mirror_peer_set_direction(
+    librados::IoCtx *ioctx, const std::string &uuid,
+    cls::rbd::MirrorPeerDirection mirror_peer_direction);
 
 void mirror_image_list_start(librados::ObjectReadOperation *op,
                              const std::string &start, uint64_t max_return);
@@ -455,11 +458,16 @@ void mirror_image_status_list_start(librados::ObjectReadOperation *op,
 int mirror_image_status_list_finish(bufferlist::const_iterator *iter,
                                     std::map<std::string, cls::rbd::MirrorImage> *images,
                                     std::map<std::string, cls::rbd::MirrorImageStatus> *statuses);
-int mirror_image_status_get_summary(librados::IoCtx *ioctx,
-                                    std::map<cls::rbd::MirrorImageStatusState, int> *states);
-void mirror_image_status_get_summary_start(librados::ObjectReadOperation *op);
-int mirror_image_status_get_summary_finish(bufferlist::const_iterator *iter,
-                                           std::map<cls::rbd::MirrorImageStatusState, int> *states);
+int mirror_image_status_get_summary(
+    librados::IoCtx *ioctx,
+    const std::vector<cls::rbd::MirrorPeer>& mirror_peer_sites,
+    std::map<cls::rbd::MirrorImageStatusState, int> *states);
+void mirror_image_status_get_summary_start(
+    librados::ObjectReadOperation *op,
+    const std::vector<cls::rbd::MirrorPeer>& mirror_peer_sites);
+int mirror_image_status_get_summary_finish(
+    bufferlist::const_iterator *iter,
+    std::map<cls::rbd::MirrorImageStatusState, int> *states);
 int mirror_image_status_remove_down(librados::IoCtx *ioctx);
 void mirror_image_status_remove_down(librados::ObjectWriteOperation *op);
 
index 44305701d5cb283a6b7d78169f7b7e252a635173..a9c3d459165d90902c5fb1bab9a52e57c0a9bdfc 100644 (file)
@@ -1499,7 +1499,7 @@ int Mirror<I>::image_status_summary(librados::IoCtx& io_ctx,
   CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 
   std::map<cls::rbd::MirrorImageStatusState, int> states_;
-  int r = cls_client::mirror_image_status_get_summary(&io_ctx, &states_);
+  int r = cls_client::mirror_image_status_get_summary(&io_ctx, {}, &states_);
   if (r < 0 && r != -ENOENT) {
     lderr(cct) << "failed to get mirror status summary: "
                << cpp_strerror(r) << dendl;
index 599b96953d22621dace364210d96ab72525fe732..8f22f54754d44a8efaaf87c0e8eaf50690f0c163 100644 (file)
@@ -1848,7 +1848,9 @@ TEST_F(TestClsRbd, mirror_image_status) {
 
   // Test status summary. All statuses are unknown due to down.
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  cls::rbd::MirrorPeer mirror_peer{
+    "uuid", cls::rbd::MIRROR_PEER_DIRECTION_RX, "siteA", "client", "fsidA"};
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(1U, states.size());
   ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
 
@@ -1867,7 +1869,7 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_EQ(0, mirror_image_status_list(&ioctx, "", 1024, &images, &statuses));
   ASSERT_EQ(3U, images.size());
   ASSERT_TRUE(statuses.empty());
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(1U, states.size());
   ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
 
@@ -1923,7 +1925,7 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_EQ(statuses["image_id3"], cls::rbd::MirrorImageStatus{{status3}});
 
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(3U, states.size());
   ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
   ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING]);
@@ -1938,42 +1940,101 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_EQ(read_status, cls::rbd::MirrorImageStatus{{status3}});
 
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(1U, states.size());
   ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING]);
 
-  // Test statuses are down after removing watcher
+  // Remote status
+
+  ASSERT_EQ(0, mirror_uuid_set(&ioctx, "mirror-uuid"));
+  ASSERT_EQ(0, mirror_mode_set(&ioctx, cls::rbd::MIRROR_MODE_POOL));
 
+  ASSERT_EQ(0, mirror_image_status_get(&ioctx, "uuid1", &read_status));
+  cls::rbd::MirrorImageStatus expected_status1({status1});
+  ASSERT_EQ(expected_status1, read_status);
+
+  cls::rbd::MirrorImageSiteStatus remote_status1(
+    "fsidA", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "");
+  ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid1", remote_status1));
+  ASSERT_EQ(0, mirror_image_status_get(&ioctx, "uuid1", &read_status));
+  remote_status1.up = true;
+  expected_status1 = {{status1, remote_status1}};
+  ASSERT_EQ(expected_status1, read_status);
+
+  // summary under different modes
+  cls::rbd::MirrorImageSiteStatus remote_status2(
+    "fsidA", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "");
+  remote_status2.up = true;
+  cls::rbd::MirrorImageSiteStatus remote_status3(
+    "fsidA", cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "");
+  remote_status3.up = true;
+
+  status1.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR;
   ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid1", status1));
   ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid2", status2));
   ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid3", status3));
+  ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid2", remote_status2));
+  ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid3", remote_status3));
+
+  expected_status1 = {{status1, remote_status1}};
+  cls::rbd::MirrorImageStatus expected_status2({status2, remote_status2});
+  cls::rbd::MirrorImageStatus expected_status3({status3, remote_status3});
 
   images.clear();
   statuses.clear();
   ASSERT_EQ(0, mirror_image_status_list(&ioctx, "", 1024, &images, &statuses));
   ASSERT_EQ(3U, images.size());
   ASSERT_EQ(3U, statuses.size());
-  ASSERT_EQ(statuses["image_id1"], cls::rbd::MirrorImageStatus{{status1}});
-  ASSERT_EQ(statuses["image_id2"], cls::rbd::MirrorImageStatus{{status2}});
-  ASSERT_EQ(statuses["image_id3"], cls::rbd::MirrorImageStatus{{status3}});
+  ASSERT_EQ(statuses["image_id1"], expected_status1);
+  ASSERT_EQ(statuses["image_id2"], expected_status2);
+  ASSERT_EQ(statuses["image_id3"], expected_status3);
+
+  states.clear();
+  mirror_peer.mirror_peer_direction = cls::rbd::MIRROR_PEER_DIRECTION_RX;
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
+  ASSERT_EQ(2U, states.size());
+  ASSERT_EQ(2, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING]);
+  ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR]);
 
+  states.clear();
+  mirror_peer.mirror_peer_direction = cls::rbd::MIRROR_PEER_DIRECTION_TX;
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
+  ASSERT_EQ(2U, states.size());
+  ASSERT_EQ(2, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING]);
+  ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
+
+  states.clear();
+  mirror_peer.mirror_peer_direction = cls::rbd::MIRROR_PEER_DIRECTION_RX_TX;
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
+  ASSERT_EQ(3U, states.size());
+  ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING]);
+  ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
+  ASSERT_EQ(1, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR]);
+
+  // Test statuses are down after removing watcher
   ioctx.unwatch2(watch_handle);
 
   ASSERT_EQ(0, mirror_image_status_list(&ioctx, "", 1024, &images, &statuses));
   ASSERT_EQ(3U, images.size());
   ASSERT_EQ(3U, statuses.size());
   status1.up = false;
-  ASSERT_EQ(statuses["image_id1"], cls::rbd::MirrorImageStatus{{status1}});
+  remote_status1.up = false;
+  expected_status1 = {{status1, remote_status1}};
+  ASSERT_EQ(statuses["image_id1"], expected_status1);
   status2.up = false;
-  ASSERT_EQ(statuses["image_id2"], cls::rbd::MirrorImageStatus{{status2}});
+  remote_status2.up = false;
+  expected_status2 = {{status2, remote_status2}};
+  ASSERT_EQ(statuses["image_id2"], expected_status2);
   status3.up = false;
-  ASSERT_EQ(statuses["image_id3"], cls::rbd::MirrorImageStatus{{status3}});
+  remote_status3.up = false;
+  expected_status3 = {{status3, remote_status3}};
+  ASSERT_EQ(statuses["image_id3"], expected_status3);
 
   ASSERT_EQ(0, mirror_image_status_get(&ioctx, "uuid1", &read_status));
-  ASSERT_EQ(read_status, cls::rbd::MirrorImageStatus{{status1}});
+  ASSERT_EQ(read_status, expected_status1);
 
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(1U, states.size());
   ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
 
@@ -1992,7 +2053,7 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_TRUE(statuses.empty());
 
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {mirror_peer}, &states));
   ASSERT_EQ(1U, states.size());
   ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]);
 
@@ -2011,7 +2072,7 @@ TEST_F(TestClsRbd, mirror_image_status) {
   ASSERT_EQ(0, mirror_image_remove(&ioctx, "image_id3"));
 
   states.clear();
-  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, &states));
+  ASSERT_EQ(0, mirror_image_status_get_summary(&ioctx, {}, &states));
   ASSERT_EQ(0U, states.size());
 
   // Test status list with large number of images