From ab410213fedc0247161809258966575688fa8cb9 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Mon, 22 Feb 2021 23:21:47 -0500 Subject: [PATCH] cephfs-mirror: use peer cluster monitor address (and key) if available This allows connecting to the peer cluster without having the cluster configuration file on the primary cluster. Signed-off-by: Venky Shankar --- src/tools/cephfs_mirror/FSMirror.cc | 2 +- src/tools/cephfs_mirror/PeerReplayer.cc | 50 +++++++++++++++++++++++-- src/tools/cephfs_mirror/PeerReplayer.h | 5 ++- src/tools/cephfs_mirror/Utils.cc | 23 ++++++++++-- src/tools/cephfs_mirror/Utils.h | 2 +- 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/tools/cephfs_mirror/FSMirror.cc b/src/tools/cephfs_mirror/FSMirror.cc index b960d60cd79..47dfe530b57 100644 --- a/src/tools/cephfs_mirror/FSMirror.cc +++ b/src/tools/cephfs_mirror/FSMirror.cc @@ -371,7 +371,7 @@ void FSMirror::add_peer(const Peer &peer) { } auto replayer = std::make_unique( - m_cct, this, m_filesystem, peer, m_directories, m_mount, m_service_daemon); + m_cct, this, m_cluster, m_filesystem, peer, m_directories, m_mount, m_service_daemon); int r = init_replayer(replayer.get()); if (r < 0) { m_service_daemon->add_or_update_peer_attribute(m_filesystem.fscid, peer, diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc index 8defdf56ba7..89368f3b86a 100644 --- a/src/tools/cephfs_mirror/PeerReplayer.cc +++ b/src/tools/cephfs_mirror/PeerReplayer.cc @@ -14,6 +14,8 @@ #include "PeerReplayer.h" #include "Utils.h" +#include "json_spirit/json_spirit.h" + #define dout_context g_ceph_context #define dout_subsys ceph_subsys_cephfs_mirror #undef dout_prefix @@ -25,6 +27,8 @@ namespace mirror { namespace { +const std::string PEER_CONFIG_KEY_PREFIX = "cephfs/mirror/peer"; + std::string snapshot_dir_path(CephContext *cct, const std::string &path) { return path + "/" + cct->_conf->client_snapdir; } @@ -51,6 +55,10 @@ std::map decode_snap_metadata(snap_metadata *snap_meta return metadata; } +std::string peer_config_key(const std::string &fs_name, const std::string &uuid) { + return PEER_CONFIG_KEY_PREFIX + "/" + fs_name + "/" + uuid; +} + class PeerAdminSocketCommand { public: virtual ~PeerAdminSocketCommand() { @@ -116,11 +124,12 @@ private: }; PeerReplayer::PeerReplayer(CephContext *cct, FSMirror *fs_mirror, - const Filesystem &filesystem, const Peer &peer, - const std::set> &directories, + RadosRef local_cluster, const Filesystem &filesystem, + const Peer &peer, const std::set> &directories, MountRef mount, ServiceDaemon *service_daemon) : m_cct(cct), m_fs_mirror(fs_mirror), + m_local_cluster(local_cluster), m_filesystem(filesystem), m_peer(peer), m_directories(directories.begin(), directories.end()), @@ -149,7 +158,42 @@ int PeerReplayer::init() { auto &remote_cluster = m_peer.remote.cluster_name; auto remote_filesystem = Filesystem{0, m_peer.remote.fs_name}; - int r = connect(remote_client, remote_cluster, &m_remote_cluster); + std::string key = peer_config_key(m_filesystem.fs_name, m_peer.uuid); + std::string cmd = + "{" + "\"prefix\": \"config-key get\", " + "\"key\": \"" + key + "\"" + "}"; + + bufferlist in_bl; + bufferlist out_bl; + + int r = m_local_cluster->mon_command(cmd, in_bl, &out_bl, nullptr); + dout(5) << ": mon command r=" << r << dendl; + if (r < 0 && r != -ENOENT) { + return r; + } + + std::string mon_host; + std::string cephx_key; + if (!r) { + json_spirit::mValue root; + if (!json_spirit::read(out_bl.to_str(), root)) { + derr << ": invalid config-key JSON" << dendl; + return -EBADMSG; + } + try { + auto &root_obj = root.get_obj(); + mon_host = root_obj.at("mon_host").get_str(); + cephx_key = root_obj.at("key").get_str(); + dout(0) << ": remote monitor host=" << mon_host << dendl; + } catch (std::runtime_error&) { + derr << ": unexpected JSON received" << dendl; + return -EBADMSG; + } + } + + r = connect(remote_client, remote_cluster, &m_remote_cluster, mon_host, cephx_key); if (r < 0) { derr << ": error connecting to remote cluster: " << cpp_strerror(r) << dendl; diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h index 3c8743ed1f9..42d58b57416 100644 --- a/src/tools/cephfs_mirror/PeerReplayer.h +++ b/src/tools/cephfs_mirror/PeerReplayer.h @@ -19,8 +19,8 @@ class PeerReplayerAdminSocketHook; class PeerReplayer { public: PeerReplayer(CephContext *cct, FSMirror *fs_mirror, - const Filesystem &filesystem, const Peer &peer, - const std::set> &directories, + RadosRef local_cluster, const Filesystem &filesystem, + const Peer &peer, const std::set> &directories, MountRef mount, ServiceDaemon *service_daemon); ~PeerReplayer(); @@ -219,6 +219,7 @@ private: CephContext *m_cct; FSMirror *m_fs_mirror; + RadosRef m_local_cluster; Filesystem m_filesystem; Peer m_peer; // probably need to be encapsulated when supporting cancelations diff --git a/src/tools/cephfs_mirror/Utils.cc b/src/tools/cephfs_mirror/Utils.cc index 65dbcbd67a8..617ae5ab4a6 100644 --- a/src/tools/cephfs_mirror/Utils.cc +++ b/src/tools/cephfs_mirror/Utils.cc @@ -18,9 +18,9 @@ namespace cephfs { namespace mirror { int connect(std::string_view client_name, std::string_view cluster_name, - RadosRef *cluster) { + RadosRef *cluster, std::string_view mon_host, std::string_view cephx_key) { dout(20) << ": connecting to cluster=" << cluster_name << ", client=" << client_name - << dendl; + << ", mon_host=" << mon_host << dendl; CephInitParameters iparams(CEPH_ENTITY_TYPE_CLIENT); if (client_name.empty() || !iparams.name.from_str(client_name)) { @@ -33,7 +33,7 @@ int connect(std::string_view client_name, std::string_view cluster_name, cct->_conf->cluster = cluster_name; int r = cct->_conf.parse_config_files(nullptr, nullptr, 0); - if (r < 0) { + if (r < 0 && r != -ENOENT) { derr << ": could not read ceph conf: " << ": " << cpp_strerror(r) << dendl; return r; } @@ -49,6 +49,23 @@ int connect(std::string_view client_name, std::string_view cluster_name, } cct->_conf.parse_env(cct->get_module_type()); + if (!mon_host.empty()) { + r = cct->_conf.set_val("mon_host", std::string(mon_host)); + if (r < 0) { + derr << "failed to set mon_host config: " << cpp_strerror(r) << dendl; + cct->put(); + return r; + } + } + if (!cephx_key.empty()) { + r = cct->_conf.set_val("key", std::string(cephx_key)); + if (r < 0) { + derr << "failed to set key config: " << cpp_strerror(r) << dendl; + cct->put(); + return r; + } + } + cluster->reset(new librados::Rados()); r = (*cluster)->init_with_context(cct); diff --git a/src/tools/cephfs_mirror/Utils.h b/src/tools/cephfs_mirror/Utils.h index c3462951a4b..837b834221e 100644 --- a/src/tools/cephfs_mirror/Utils.h +++ b/src/tools/cephfs_mirror/Utils.h @@ -10,7 +10,7 @@ namespace cephfs { namespace mirror { int connect(std::string_view client_name, std::string_view cluster_name, - RadosRef *cluster); + RadosRef *cluster, std::string_view mon_host={}, std::string_view cephx_key={}); int mount(RadosRef cluster, const Filesystem &filesystem, bool cross_check_fscid, MountRef *mount); -- 2.39.5