cephfs-mirror: use peer cluster monitor address (and key) if available

This allows connecting to the peer cluster without having the cluster
configuration file on the primary cluster.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit ab410213fedc0247161809258966575688fa8cb9)

diff --git a/src/tools/cephfs_mirror/FSMirror.cc b/src/tools/cephfs_mirror/FSMirror.cc
index b960d60cd7950d8e37939c04f825f4562a3e510d..47dfe530b57c61e59e79b54aa6ee370e68bec012 100644 (file)
--- a/src/tools/cephfs_mirror/FSMirror.cc
+++ b/src/tools/cephfs_mirror/FSMirror.cc
@@ -371,7 +371,7 @@ void FSMirror::add_peer(const Peer &peer) {
   }
 
   auto replayer = std::make_unique<PeerReplayer>(
-    m_cct, this, m_filesystem, peer, m_directories, m_mount, m_service_daemon);
+    m_cct, this, m_cluster, m_filesystem, peer, m_directories, m_mount, m_service_daemon);
   int r = init_replayer(replayer.get());
   if (r < 0) {
     m_service_daemon->add_or_update_peer_attribute(m_filesystem.fscid, peer,

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index 8defdf56ba7d975f30bb3ccb2ab8f1bb407ba262..89368f3b86aa4d287f2ca341be4d99972894c9f4 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -14,6 +14,8 @@
 #include "PeerReplayer.h"
 #include "Utils.h"
 
+#include "json_spirit/json_spirit.h"
+
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_cephfs_mirror
 #undef dout_prefix
@@ -25,6 +27,8 @@ namespace mirror {
 
 namespace {
 
+const std::string PEER_CONFIG_KEY_PREFIX = "cephfs/mirror/peer";
+
 std::string snapshot_dir_path(CephContext *cct, const std::string &path) {
   return path + "/" + cct->_conf->client_snapdir;
 }
@@ -51,6 +55,10 @@ std::map<std::string, std::string> decode_snap_metadata(snap_metadata *snap_meta
   return metadata;
 }
 
+std::string peer_config_key(const std::string &fs_name, const std::string &uuid) {
+  return PEER_CONFIG_KEY_PREFIX + "/" + fs_name + "/" + uuid;
+}
+
 class PeerAdminSocketCommand {
 public:
   virtual ~PeerAdminSocketCommand() {
@@ -116,11 +124,12 @@ private:
 };
 
 PeerReplayer::PeerReplayer(CephContext *cct, FSMirror *fs_mirror,
-                           const Filesystem &filesystem, const Peer &peer,
-                           const std::set<std::string, std::less<>> &directories,
+                           RadosRef local_cluster, const Filesystem &filesystem,
+                           const Peer &peer, const std::set<std::string, std::less<>> &directories,
                            MountRef mount, ServiceDaemon *service_daemon)
   : m_cct(cct),
     m_fs_mirror(fs_mirror),
+    m_local_cluster(local_cluster),
     m_filesystem(filesystem),
     m_peer(peer),
     m_directories(directories.begin(), directories.end()),
@@ -149,7 +158,42 @@ int PeerReplayer::init() {
   auto &remote_cluster = m_peer.remote.cluster_name;
   auto remote_filesystem = Filesystem{0, m_peer.remote.fs_name};
 
-  int r = connect(remote_client, remote_cluster, &m_remote_cluster);
+  std::string key = peer_config_key(m_filesystem.fs_name, m_peer.uuid);
+  std::string cmd =
+    "{"
+      "\"prefix\": \"config-key get\", "
+      "\"key\": \"" + key + "\""
+    "}";
+
+  bufferlist in_bl;
+  bufferlist out_bl;
+
+  int r = m_local_cluster->mon_command(cmd, in_bl, &out_bl, nullptr);
+  dout(5) << ": mon command r=" << r << dendl;
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+
+  std::string mon_host;
+  std::string cephx_key;
+  if (!r) {
+    json_spirit::mValue root;
+    if (!json_spirit::read(out_bl.to_str(), root)) {
+      derr << ": invalid config-key JSON" << dendl;
+      return -EBADMSG;
+    }
+    try {
+      auto &root_obj = root.get_obj();
+      mon_host = root_obj.at("mon_host").get_str();
+      cephx_key = root_obj.at("key").get_str();
+      dout(0) << ": remote monitor host=" << mon_host << dendl;
+    } catch (std::runtime_error&) {
+      derr << ": unexpected JSON received" << dendl;
+      return -EBADMSG;
+    }
+  }
+
+  r = connect(remote_client, remote_cluster, &m_remote_cluster, mon_host, cephx_key);
   if (r < 0) {
     derr << ": error connecting to remote cluster: " << cpp_strerror(r)
          << dendl;

diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 3c8743ed1f9cdc67632a22915c8f58183d30ff3d..42d58b574168f3682271981c819f7c3ff15caf77 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -19,8 +19,8 @@ class PeerReplayerAdminSocketHook;
 class PeerReplayer {
 public:
   PeerReplayer(CephContext *cct, FSMirror *fs_mirror,
-               const Filesystem &filesystem, const Peer &peer,
-               const std::set<std::string, std::less<>> &directories,
+               RadosRef local_cluster, const Filesystem &filesystem,
+               const Peer &peer, const std::set<std::string, std::less<>> &directories,
                MountRef mount, ServiceDaemon *service_daemon);
   ~PeerReplayer();
 
@@ -219,6 +219,7 @@ private:
 
   CephContext *m_cct;
   FSMirror *m_fs_mirror;
+  RadosRef m_local_cluster;
   Filesystem m_filesystem;
   Peer m_peer;
   // probably need to be encapsulated when supporting cancelations

diff --git a/src/tools/cephfs_mirror/Utils.cc b/src/tools/cephfs_mirror/Utils.cc
index 65dbcbd67a84c2ac0d4cac58705fd890d2fab778..617ae5ab4a61e624dcc2e061f2655e814d52305f 100644 (file)
--- a/src/tools/cephfs_mirror/Utils.cc
+++ b/src/tools/cephfs_mirror/Utils.cc
@@ -18,9 +18,9 @@ namespace cephfs {
 namespace mirror {
 
 int connect(std::string_view client_name, std::string_view cluster_name,
-            RadosRef *cluster) {
+            RadosRef *cluster, std::string_view mon_host, std::string_view cephx_key) {
   dout(20) << ": connecting to cluster=" << cluster_name << ", client=" << client_name
-           << dendl;
+           << ", mon_host=" << mon_host << dendl;
 
   CephInitParameters iparams(CEPH_ENTITY_TYPE_CLIENT);
   if (client_name.empty() || !iparams.name.from_str(client_name)) {
@@ -33,7 +33,7 @@ int connect(std::string_view client_name, std::string_view cluster_name,
   cct->_conf->cluster = cluster_name;
 
   int r = cct->_conf.parse_config_files(nullptr, nullptr, 0);
-  if (r < 0) {
+  if (r < 0 && r != -ENOENT) {
     derr << ": could not read ceph conf: " << ": " << cpp_strerror(r) << dendl;
     return r;
   }
@@ -49,6 +49,23 @@ int connect(std::string_view client_name, std::string_view cluster_name,
   }
   cct->_conf.parse_env(cct->get_module_type());
 
+  if (!mon_host.empty()) {
+    r = cct->_conf.set_val("mon_host", std::string(mon_host));
+    if (r < 0) {
+      derr << "failed to set mon_host config: " << cpp_strerror(r) << dendl;
+      cct->put();
+      return r;
+    }
+  }
+  if (!cephx_key.empty()) {
+    r = cct->_conf.set_val("key", std::string(cephx_key));
+    if (r < 0) {
+      derr << "failed to set key config: " << cpp_strerror(r) << dendl;
+      cct->put();
+      return r;
+    }
+  }
+
   cluster->reset(new librados::Rados());
 
   r = (*cluster)->init_with_context(cct);

diff --git a/src/tools/cephfs_mirror/Utils.h b/src/tools/cephfs_mirror/Utils.h
index c3462951a4b8d6915bf541d8f956ef2f798cc9d3..837b834221eaf13436c9b46ce6e439aac13af9a1 100644 (file)
--- a/src/tools/cephfs_mirror/Utils.h
+++ b/src/tools/cephfs_mirror/Utils.h
@@ -10,7 +10,7 @@ namespace cephfs {
 namespace mirror {
 
 int connect(std::string_view client_name, std::string_view cluster_name,
-            RadosRef *cluster);
+            RadosRef *cluster, std::string_view mon_host={}, std::string_view cephx_key={});
 
 int mount(RadosRef cluster, const Filesystem &filesystem, bool cross_check_fscid,
           MountRef *mount);