mds: track filesystem mirror peers in fsmap

There is a need to track mirror peers for a ceph filesystem
to be used by cephfs-mirror daemons for synchronizing data
(snapshots) to its peers.

One approach would have been to persist peer information
(remote cluster name and user {client-id}) in RADOS, possibly
in omap, and have cephfs-mirror daemons periodically scan for
peer changes. Another approach is to track filesystem mirror
peers in FSMap. This makes cephfs-mirror daemon receive peer
updates and does away with any kind of repetitive scans.
Also, this makes peer information available to FSMap
subscribers to take action based on list of peers.

Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index 9fd18f4e187c356ece1a09366ea77f94437bc4e6..4c1fce12b6b8dc43f76266ba0a7950e4d46cb36e 100644 (file)
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -36,12 +36,94 @@ using std::stringstream;
 using ceph::bufferlist;
 using ceph::Formatter;
 
+void ClusterInfo::encode(ceph::buffer::list &bl) const {
+  ENCODE_START(1, 1, bl);
+  encode(client_name, bl);
+  encode(cluster_name, bl);
+  encode(fs_name, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ClusterInfo::decode(ceph::buffer::list::const_iterator &iter) {
+  DECODE_START(1, iter);
+  decode(client_name, iter);
+  decode(cluster_name, iter);
+  decode(fs_name, iter);
+  DECODE_FINISH(iter);
+}
+
+void ClusterInfo::dump(ceph::Formatter *f) const {
+  f->dump_string("client_name", client_name);
+  f->dump_string("cluster_name", cluster_name);
+  f->dump_string("fs_name", fs_name);
+}
+
+void ClusterInfo::print(std::ostream& out) const {
+  out << "[client_name=" << client_name << ", cluster_name=" << cluster_name
+      << ", fs_name=" << fs_name << "]" << std::endl;
+}
+
+void Peer::encode(ceph::buffer::list &bl) const {
+  ENCODE_START(1, 1, bl);
+  encode(uuid, bl);
+  encode(remote, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Peer::decode(ceph::buffer::list::const_iterator &iter) {
+  DECODE_START(1, iter);
+  decode(uuid, iter);
+  decode(remote, iter);
+  DECODE_FINISH(iter);
+}
+
+void Peer::dump(ceph::Formatter *f) const {
+  f->open_object_section(uuid);
+  f->dump_object("remote", remote);
+  f->close_section();
+}
+
+void Peer::print(std::ostream& out) const {
+  out << "[uuid=" << uuid << ", remote=" << remote << "]" << std::endl;
+}
+
+void MirrorInfo::encode(ceph::buffer::list &bl) const {
+  ENCODE_START(1, 1, bl);
+  encode(mirrored, bl);
+  encode(peers, bl);
+  ENCODE_FINISH(bl);
+}
+
+void MirrorInfo::decode(ceph::buffer::list::const_iterator &iter) {
+  DECODE_START(1, iter);
+  decode(mirrored, iter);
+  decode(peers, iter);
+  DECODE_FINISH(iter);
+}
+
+void MirrorInfo::dump(ceph::Formatter *f) const {
+  f->open_object_section("peers");
+  for (auto &peer : peers) {
+    peer.dump(f);
+  }
+  f->close_section(); // peers
+}
+
+void MirrorInfo::print(std::ostream& out) const {
+  out << "[peers=" << peers << "]" << std::endl;
+}
+
 void Filesystem::dump(Formatter *f) const
 {
   f->open_object_section("mdsmap");
   mds_map.dump(f);
   f->close_section();
   f->dump_int("id", fscid);
+  if (mirror_info.is_mirrored()) {
+    f->open_object_section("mirror_info");
+    mirror_info.dump(f);
+    f->close_section(); // mirror_info
+  }
 }
 
 void FSMap::dump(Formatter *f) const
@@ -694,22 +776,26 @@ void FSMap::sanitize(const std::function<bool(int64_t pool)>& pool_exists)
 
 void Filesystem::encode(bufferlist& bl, uint64_t features) const
 {
-  ENCODE_START(1, 1, bl);
+  ENCODE_START(2, 1, bl);
   encode(fscid, bl);
   bufferlist mdsmap_bl;
   mds_map.encode(mdsmap_bl, features);
   encode(mdsmap_bl, bl);
+  encode(mirror_info, bl);
   ENCODE_FINISH(bl);
 }
 
 void Filesystem::decode(bufferlist::const_iterator& p)
 {
-  DECODE_START(1, p);
+  DECODE_START(2, p);
   decode(fscid, p);
   bufferlist mdsmap_bl;
   decode(mdsmap_bl, p);
   auto mdsmap_bl_iter = mdsmap_bl.cbegin();
   mds_map.decode(mdsmap_bl_iter);
+  if (struct_v >= 2) {
+    decode(mirror_info, p);
+  }
   DECODE_FINISH(p);
 }
 
@@ -740,6 +826,9 @@ void Filesystem::print(std::ostream &out) const
   out << "Filesystem '" << mds_map.fs_name
       << "' (" << fscid << ")" << std::endl;
   mds_map.print(out);
+  if (mirror_info.is_mirrored()) {
+    mirror_info.print(out);
+  }
 }
 
 bool FSMap::is_any_degraded() const

diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h
index 2a80a9891acd0c2a679958ed97487f4e7429a604..d8b496d58eebbe68e82a15e50583735dab21ac25 100644 (file)
--- a/src/mds/FSMap.h
+++ b/src/mds/FSMap.h
@@ -36,6 +36,136 @@
 
 class health_check_map_t;
 
+struct ClusterInfo {
+  ClusterInfo() = default;
+  ClusterInfo(std::string_view client_name, std::string_view cluster_name,
+              std::string_view fs_name)
+    : client_name(client_name),
+      cluster_name(cluster_name),
+      fs_name(fs_name) {
+  }
+
+  std::string client_name;
+  std::string cluster_name;
+  std::string fs_name;
+
+  bool operator==(const ClusterInfo &cluster_info) const {
+    return client_name == cluster_info.client_name &&
+           cluster_name == cluster_info.cluster_name &&
+           fs_name == cluster_info.fs_name;
+  }
+
+  void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const;
+
+  void encode(ceph::buffer::list &bl) const;
+  void decode(ceph::buffer::list::const_iterator &iter);
+};
+
+inline std::ostream& operator<<(std::ostream& out, const ClusterInfo &cluster_info) {
+  out << "{client_name=" << cluster_info.client_name << ", cluster_name="
+      << cluster_info.cluster_name << ", fs_name=" << cluster_info.fs_name << "}";
+  return out;
+}
+
+struct Peer {
+  Peer() = default;
+  Peer(std::string_view uuid)
+    : uuid(uuid) {
+  }
+  Peer(std::string_view uuid,
+       const ClusterInfo &remote)
+    : uuid(uuid),
+      remote(remote) {
+  }
+
+  std::string uuid;
+  ClusterInfo remote;
+
+  bool operator==(const Peer &rhs) const {
+    return uuid == rhs.uuid;
+  }
+
+  bool operator<(const Peer &rhs) const {
+    return uuid < rhs.uuid;
+  }
+
+  void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const;
+
+  void encode(ceph::buffer::list &bl) const;
+  void decode(ceph::buffer::list::const_iterator &iter);
+};
+
+typedef std::set<Peer> Peers;
+inline std::ostream& operator<<(std::ostream& out, const Peer &peer) {
+  out << "{uuid=" << peer.uuid << ", remote_cluster=" << peer.remote << "}";
+  return out;
+}
+
+struct MirrorInfo {
+  MirrorInfo() = default;
+
+  bool is_mirrored() const {
+    return mirrored;
+  }
+  void enable_mirroring() {
+    mirrored = true;
+  }
+  void disable_mirroring() {
+    peers.clear();
+    mirrored = false;
+  }
+
+  // uuid variant check
+  bool has_peer(std::string_view uuid) const {
+    return peers.find(Peer(uuid)) != peers.end();
+  }
+  // client_name/cluster_name/fs_name variant check
+  bool has_peer(std::string_view client_name,
+                std::string_view cluster_name,
+                std::string_view fs_name) const {
+    ClusterInfo cluster_info(client_name, cluster_name, fs_name);
+    for (auto &peer : peers) {
+      if (peer.remote == cluster_info) {
+        return true;
+      }
+    }
+    return false;
+  }
+  bool has_peers() const {
+    return !peers.empty();
+  }
+
+  void peer_add(std::string_view uuid,
+                std::string_view client_name,
+                std::string_view cluster_name,
+                std::string_view fs_name) {
+    peers.emplace(Peer(uuid, ClusterInfo(client_name, cluster_name, fs_name)));
+  }
+  void peer_remove(std::string_view uuid) {
+    peers.erase(uuid);
+  }
+
+  bool mirrored = false;
+  Peers peers;
+
+  void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const;
+
+  void encode(ceph::buffer::list &bl) const;
+  void decode(ceph::buffer::list::const_iterator &iter);
+};
+
+inline std::ostream& operator<<(std::ostream& out, const MirrorInfo &mirror_info) {
+  out << "{peers=" << mirror_info.peers << "}";
+  return out;
+}
+
+WRITE_CLASS_ENCODER(ClusterInfo)
+WRITE_CLASS_ENCODER(Peer)
+WRITE_CLASS_ENCODER(MirrorInfo)
+
 /**
  * The MDSMap and any additional fields describing a particular
  * filesystem (a unique fs_cluster_id_t).
@@ -79,6 +209,7 @@ public:
 
   fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
   MDSMap mds_map;
+  MirrorInfo mirror_info;
 };
 WRITE_CLASS_ENCODER_FEATURES(Filesystem)
 

diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 86efed8e64e52b60acf8c91d08be4a9154a4e95e..474c8ab4faf16f49e928bf3b7822147cce553de4 100644 (file)
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -1053,6 +1053,208 @@ class AliasHandler : public T
   }
 };
 
+class MirrorHandlerEnable : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerEnable()
+    : FileSystemCommandHandler("fs mirror enable")
+  {}
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::stringstream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (fs->mirror_info.is_mirrored()) {
+      return 0;
+    }
+
+    auto f = [](auto &&fs) {
+               fs->mirror_info.enable_mirroring();
+    };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+
+    return 0;
+  }
+};
+
+class MirrorHandlerDisable : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerDisable()
+    : FileSystemCommandHandler("fs mirror disable")
+  {}
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::stringstream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      return 0;
+    }
+
+    auto f = [](auto &&fs) {
+      fs->mirror_info.disable_mirroring();
+    };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+
+    return 0;
+  }
+};
+
+class MirrorHandlerAddPeer : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerAddPeer()
+    : FileSystemCommandHandler("fs mirror peer_add")
+  {}
+
+  boost::optional<std::pair<string, string>>
+  extract_remote_cluster_conf(const std::string &spec) {
+    auto pos = spec.find("@");
+    if (pos == std::string_view::npos) {
+      return boost::optional<std::pair<string, string>>();
+    }
+
+    auto client = spec.substr(0, pos);
+    auto cluster = spec.substr(pos+1);
+
+    return std::make_pair(client, cluster);
+  }
+
+  bool peer_add(FSMap &fsmap, Filesystem::const_ref &&fs,
+                const cmdmap_t &cmdmap, std::stringstream &ss) {
+    string remote_spec;
+    string remote_fs_name;
+    cmd_getval(cmdmap, "remote_cluster_spec", remote_spec);
+    cmd_getval(cmdmap, "remote_fs_name", remote_fs_name);
+
+    // verify (and extract) remote cluster specification
+    auto remote_conf = extract_remote_cluster_conf(remote_spec);
+    if (!remote_conf) {
+      ss << "invalid remote cluster spec -- should be <client>@<cluster>";
+      return false;
+    }
+
+    if (fs->mirror_info.has_peer((*remote_conf).first,
+                                 (*remote_conf).second, remote_fs_name)) {
+      ss << "peer already exists";
+      return true;
+    }
+
+    uuid_d uuid_gen;
+    uuid_gen.generate_random();
+
+    auto f = [uuid_gen, remote_conf, remote_fs_name](auto &&fs) {
+               fs->mirror_info.peer_add(stringify(uuid_gen), (*remote_conf).first,
+                                        (*remote_conf).second, remote_fs_name);
+             };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+    return true;
+  }
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::stringstream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      ss << "Mirroring not enabled for filesystem '" << fs_name << "'";
+      return -EINVAL;
+    }
+
+    auto res = peer_add(fsmap, std::move(fs), cmdmap, ss);
+    if (!res) {
+      return -EINVAL;
+    }
+
+    return 0;
+  }
+};
+
+class MirrorHandlerRemovePeer : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerRemovePeer()
+    : FileSystemCommandHandler("fs mirror peer_remove")
+  {}
+
+  bool peer_remove(FSMap &fsmap, Filesystem::const_ref &&fs,
+                   const cmdmap_t &cmdmap, std::stringstream &ss) {
+    string peer_uuid;
+    cmd_getval(cmdmap, "uuid", peer_uuid);
+
+    if (!fs->mirror_info.has_peer(peer_uuid)) {
+      ss << "cannot find peer with uuid: " << peer_uuid;
+      return true;
+    }
+
+    auto f = [peer_uuid](auto &&fs) {
+               fs->mirror_info.peer_remove(peer_uuid);
+             };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+    return true;
+  }
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::stringstream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      ss << "Mirroring not enabled for filesystem '" << fs_name << "'";
+      return -EINVAL;
+    }
+
+    auto res = peer_remove(fsmap, std::move(fs), cmdmap, ss);
+    if (!res) {
+      return -EINVAL;
+    }
+
+    return 0;
+  }
+};
 
 std::list<std::shared_ptr<FileSystemCommandHandler> >
 FileSystemCommandHandler::load(Paxos *paxos)
@@ -1072,6 +1274,10 @@ FileSystemCommandHandler::load(Paxos *paxos)
   handlers.push_back(std::make_shared<SetDefaultHandler>());
   handlers.push_back(std::make_shared<AliasHandler<SetDefaultHandler> >(
         "fs set_default"));
+  handlers.push_back(std::make_shared<MirrorHandlerEnable>());
+  handlers.push_back(std::make_shared<MirrorHandlerDisable>());
+  handlers.push_back(std::make_shared<MirrorHandlerAddPeer>());
+  handlers.push_back(std::make_shared<MirrorHandlerRemovePeer>());
 
   return handlers;
 }

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index f1e429aef23a05b4fad1d7dc0a82b805f2cab143..b194a406ff8666c3935546e7eb5d28052ac6188a 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -429,6 +429,21 @@ COMMAND_WITH_FLAG("fs set_default name=fs_name,type=CephString",
 COMMAND("fs set-default name=fs_name,type=CephString",
        "set the default to the named filesystem",
        "fs", "rw")
+COMMAND("fs mirror enable "
+       "name=fs_name,type=CephString ",
+       "enable mirroring for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror disable "
+       "name=fs_name,type=CephString ",
+       "disable mirroring for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror peer_add "
+       "name=fs_name,type=CephString "
+       "name=remote_cluster_spec,type=CephString "
+       "name=remote_fs_name,type=CephString",
+       "add a mirror peer for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror peer_remove "
+       "name=fs_name,type=CephString "
+       "name=uuid,type=CephString ",
+       "remove a mirror peer for a ceph filesystem", "mds", "rw")
 
 /*
  * Monmap commands