From 2f075704073ff80f94c70cf79516028d2754ae4f Mon Sep 17 00:00:00 2001 From: Shyamsundar Ranganathan Date: Sun, 9 Aug 2020 20:47:43 -0400 Subject: [PATCH] mon: store mon updates in ceph context for future MonMap instantiation MonMap builds initial mon list using provided sources, like mon-host or monmap. For future instantiations of MonClient, if mon addresses are updated, stale information from the provided sources are used. This commit retains mon updates that are processed by the MonClient in CephContext, for use in MonMap instantiations and hence uses updated information as required. This is helpful in cases where librados or libcephfs instantiate MonClient in the ceph-mgr deamon as required. Fixes: https://tracker.ceph.com/issues/46645 Signed-off-by: Shyamsundar Ranganathan (cherry picked from commit 7a1f02acfe6b5d8a760efd16bb594a0656b39eac) Conflicts: src/mon/MonMap.h Retain crimson namespace as ceph::common src/mon/MonMap.cc Address merge conflict due to linespace removed src/common/ceph_context.cc Remove WITH_ALIEN latch src/common/ceph_context/h Remove WITH_ALIEN latch --- src/common/ceph_context.cc | 12 ++++ src/common/ceph_context.h | 19 ++++++ src/mon/MonClient.cc | 2 + src/mon/MonMap.cc | 63 ++++++++++---------- src/mon/MonMap.h | 16 ++++- src/test/libcephfs/CMakeLists.txt | 1 + src/test/libcephfs/monconfig.cc | 99 +++++++++++++++++++++++++++++++ 7 files changed, 179 insertions(+), 33 deletions(-) create mode 100644 src/test/libcephfs/monconfig.cc diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index c2d3d28e1ece5..545e7725901f9 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -42,6 +42,9 @@ #include "common/PluginRegistry.h" #include "common/valgrind.h" #include "include/spinlock.h" +#ifndef WITH_SEASTAR +#include "mon/MonMap.h" +#endif using ceph::bufferlist; using ceph::HeartbeatMap; @@ -951,4 +954,13 @@ void CephContext::notify_post_fork() for (auto &&t : _fork_watchers) t->handle_post_fork(); } + +void CephContext::set_mon_addrs(const MonMap& mm) { + std::vector mon_addrs; + for (auto& i : mm.mon_info) { + mon_addrs.push_back(i.second.public_addrs); + } + + set_mon_addrs(mon_addrs); +} #endif // WITH_SEASTAR diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index a124059e5e8ba..7fdf324cc750f 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -29,6 +29,7 @@ #include "common/cmdparse.h" #include "common/code_environment.h" +#include "msg/msg_types.h" #ifdef WITH_SEASTAR #include "crimson/common/config_proxy.h" #include "crimson/common/perf_counters_collection.h" @@ -47,6 +48,7 @@ class CephContextHook; class CephContextObs; class CryptoHandler; class CryptoRandom; +class MonMap; namespace ceph { class PluginRegistry; @@ -242,6 +244,21 @@ public: void notify_pre_fork(); void notify_post_fork(); + /** + * update CephContext with a copy of the passed in MonMap mon addrs + * + * @param mm MonMap to extract and update mon addrs + */ + void set_mon_addrs(const MonMap& mm); + void set_mon_addrs(const std::vector& in) { + auto ptr = std::make_shared>(in); + atomic_store_explicit(&_mon_addrs, std::move(ptr), std::memory_order_relaxed); + } + std::shared_ptr> get_mon_addrs() const { + auto ptr = atomic_load_explicit(&_mon_addrs, std::memory_order_relaxed); + return ptr; + } + private: @@ -259,6 +276,8 @@ private: int _crypto_inited; + std::shared_ptr> _mon_addrs; + /* libcommon service thread. * SIGHUP wakes this thread, which then reopens logfiles */ friend class CephContextServiceThread; diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index ab24689a1cd64..c8c182aaa91c0 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -402,6 +402,8 @@ void MonClient::handle_monmap(MMonMap *m) } } + cct->set_mon_addrs(monmap); + sub.got("monmap", monmap.get_epoch()); map_cond.Signal(); want_monmap = false; diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc index 687c3f8691060..9ea47ffef4f23 100644 --- a/src/mon/MonMap.cc +++ b/src/mon/MonMap.cc @@ -449,9 +449,29 @@ void MonMap::_add_ambiguous_addr(const string& name, } } +int +MonMap::init_with_addrs(const std::vector& addrs, + bool for_mkfs, + std::string_view prefix) +{ + char id = 'a'; + for (auto& addr : addrs) { + string name{prefix}; + name += id++; + if (addr.v.size() == 1) { + _add_ambiguous_addr(name, addr.front(), 0, for_mkfs); + } else { + // they specified an addrvec, so let's assume they also specified + // the addr *type* and *port*. (we could possibly improve this?) + add(name, addr, 0); + } + } + return 0; +} + int MonMap::init_with_ips(const std::string& ips, bool for_mkfs, - const std::string &prefix) + std::string_view prefix) { vector addrs; if (!parse_ip_port_vec( @@ -461,27 +481,12 @@ int MonMap::init_with_ips(const std::string& ips, } if (addrs.empty()) return -ENOENT; - for (unsigned i=0; i_conf; + + // cct? + auto addrs = cct->get_mon_addrs(); + if (addrs != nullptr && (addrs->size() > 0)) { + return init_with_addrs(*addrs, for_mkfs, "noname-"); + } + // file? if (const auto monmap = conf.get_val("monmap"); !monmap.empty()) { diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h index 47225aa79460e..6b82bd10cc55b 100644 --- a/src/mon/MonMap.h +++ b/src/mon/MonMap.h @@ -436,6 +436,18 @@ public: static void generate_test_instances(list& o); protected: + /** + * build a monmap from a list of entity_addrvec_t's + * + * Give mons dummy names. + * + * @param addrs list of entity_addrvec_t's + * @param prefix prefix to prepend to generated mon names + * @return 0 for success, -errno on error + */ + int init_with_addrs(const std::vector& addrs, + bool for_mkfs, + std::string_view prefix); /** * build a monmap from a list of ips * @@ -447,7 +459,7 @@ protected: */ int init_with_ips(const std::string& ips, bool for_mkfs, - const std::string &prefix); + std::string_view prefix); /** * build a monmap from a list of hostnames * @@ -459,7 +471,7 @@ protected: */ int init_with_hosts(const std::string& hostlist, bool for_mkfs, - const std::string& prefix); + std::string_view prefix); int init_with_config_file(const ConfigProxy& conf, std::ostream& errout); #if WITH_SEASTAR seastar::future<> read_monmap(const std::string& monmap); diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt index f093324e7941f..4ede0ecaa954a 100644 --- a/src/test/libcephfs/CMakeLists.txt +++ b/src/test/libcephfs/CMakeLists.txt @@ -9,6 +9,7 @@ if(${WITH_CEPHFS}) acl.cc main.cc deleg.cc + monconfig.cc ) target_link_libraries(ceph_test_libcephfs ceph-common diff --git a/src/test/libcephfs/monconfig.cc b/src/test/libcephfs/monconfig.cc new file mode 100644 index 0000000000000..c962c4f3ab02e --- /dev/null +++ b/src/test/libcephfs/monconfig.cc @@ -0,0 +1,99 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gtest/gtest.h" +#include "include/cephfs/libcephfs.h" +#include "common/ceph_context.h" +#include +#include +#include +#include +#include + +class MonConfig : public ::testing::Test +{ + protected: + struct ceph_mount_info *ca; + + void SetUp() override { + ASSERT_EQ(0, ceph_create(&ca, NULL)); + ASSERT_EQ(0, ceph_conf_read_file(ca, NULL)); + ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL)); + } + + void TearDown() override { + ceph_shutdown(ca); + } + + // Helper to remove/unset all possible mon information from ConfigProxy + void clear_mon_config(CephContext *cct) { + auto& conf = cct->_conf; + // Clear safe_to_start_threads, allowing updates to config values + conf._clear_safe_to_start_threads(); + ASSERT_EQ(0, conf.set_val("monmap", "", nullptr)); + ASSERT_EQ(0, conf.set_val("mon_host", "", nullptr)); + ASSERT_EQ(0, conf.set_val("mon_dns_srv_name", "", nullptr)); + conf.set_safe_to_start_threads(); + } + + // Helper to test basic operation on a mount + void use_mount(struct ceph_mount_info *mnt, string name_prefix) { + char name[20]; + snprintf(name, sizeof(name), "%s.%d", name_prefix.c_str(), getpid()); + int fd = ceph_open(mnt, name, O_CREAT|O_RDWR, 0644); + ASSERT_LE(0, fd); + + ceph_close(mnt, fd); + } +}; + +TEST_F(MonConfig, MonAddrsMissing) { + CephContext *cct; + + // Test mount failure when there is no known mon config source + cct = ceph_get_mount_context(ca); + ASSERT_NE(nullptr, cct); + clear_mon_config(cct); + + ASSERT_EQ(-ENOENT, ceph_mount(ca, NULL)); +} + +TEST_F(MonConfig, MonAddrsInConfigProxy) { + // Test a successful mount with default mon config source in ConfigProxy + ASSERT_EQ(0, ceph_mount(ca, NULL)); + + use_mount(ca, "foo"); +} + +TEST_F(MonConfig, MonAddrsInCct) { + struct ceph_mount_info *cb; + CephContext *cct; + + // Perform mount to bootstrap mon addrs in CephContext + ASSERT_EQ(0, ceph_mount(ca, NULL)); + + // Reuse bootstrapped CephContext, clearing ConfigProxy mon addr sources + cct = ceph_get_mount_context(ca); + ASSERT_NE(nullptr, cct); + clear_mon_config(cct); + ASSERT_EQ(0, ceph_create_with_context(&cb, cct)); + + // Test a successful mount with only mon values in CephContext + ASSERT_EQ(0, ceph_mount(cb, NULL)); + + use_mount(ca, "bar"); + use_mount(cb, "bar"); + + ceph_shutdown(cb); +} -- 2.39.5