From 7a1f02acfe6b5d8a760efd16bb594a0656b39eac Mon Sep 17 00:00:00 2001 From: Shyamsundar Ranganathan Date: Sun, 9 Aug 2020 20:47:43 -0400 Subject: [PATCH] mon: store mon updates in ceph context for future MonMap instantiation MonMap builds initial mon list using provided sources, like mon-host or monmap. For future instantiations of MonClient, if mon addresses are updated, stale information from the provided sources are used. This commit retains mon updates that are processed by the MonClient in CephContext, for use in MonMap instantiations and hence uses updated information as required. This is helpful in cases where librados or libcephfs instantiate MonClient in the ceph-mgr deamon as required. Fixes: https://tracker.ceph.com/issues/46645 Signed-off-by: Shyamsundar Ranganathan --- src/common/ceph_context.cc | 12 ++++ src/common/ceph_context.h | 19 ++++++ src/mon/MonClient.cc | 2 + src/mon/MonMap.cc | 64 ++++++++++---------- src/mon/MonMap.h | 18 +++++- src/test/libcephfs/CMakeLists.txt | 1 + src/test/libcephfs/monconfig.cc | 99 +++++++++++++++++++++++++++++++ 7 files changed, 180 insertions(+), 35 deletions(-) create mode 100644 src/test/libcephfs/monconfig.cc diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index 2e46507955c7..e095bbdca4bd 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -45,6 +45,9 @@ #include "common/PluginRegistry.h" #include "common/valgrind.h" #include "include/spinlock.h" +#if !(defined(WITH_SEASTAR) && !defined(WITH_ALIEN)) +#include "mon/MonMap.h" +#endif // for CINIT_FLAGS #include "common/common_init.h" @@ -991,5 +994,14 @@ void CephContext::notify_post_fork() for (auto &&t : _fork_watchers) t->handle_post_fork(); } + +void CephContext::set_mon_addrs(const MonMap& mm) { + std::vector mon_addrs; + for (auto& i : mm.mon_info) { + mon_addrs.push_back(i.second.public_addrs); + } + + set_mon_addrs(mon_addrs); +} } #endif // WITH_SEASTAR diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 3d1cff123c7e..27f769007842 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -33,6 +33,7 @@ #include "common/cmdparse.h" #include "common/code_environment.h" +#include "msg/msg_types.h" #if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #include "crimson/common/config_proxy.h" #include "crimson/common/perf_counters_collection.h" @@ -48,6 +49,7 @@ class AdminSocket; class CryptoHandler; class CryptoRandom; +class MonMap; namespace ceph::common { class CephContextServiceThread; @@ -261,6 +263,21 @@ public: void notify_pre_fork(); void notify_post_fork(); + /** + * update CephContext with a copy of the passed in MonMap mon addrs + * + * @param mm MonMap to extract and update mon addrs + */ + void set_mon_addrs(const MonMap& mm); + void set_mon_addrs(const std::vector& in) { + auto ptr = std::make_shared>(in); + atomic_store_explicit(&_mon_addrs, std::move(ptr), std::memory_order_relaxed); + } + std::shared_ptr> get_mon_addrs() const { + auto ptr = atomic_load_explicit(&_mon_addrs, std::memory_order_relaxed); + return ptr; + } + private: @@ -278,6 +295,8 @@ private: int _crypto_inited; + std::shared_ptr> _mon_addrs; + /* libcommon service thread. * SIGHUP wakes this thread, which then reopens logfiles */ friend class CephContextServiceThread; diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index de51bc70416c..a9a63243d331 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -424,6 +424,8 @@ void MonClient::handle_monmap(MMonMap *m) } } + cct->set_mon_addrs(monmap); + sub.got("monmap", monmap.get_epoch()); map_cond.notify_all(); want_monmap = false; diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc index dc9c4a8b1a5d..01c4ab8fd8a8 100644 --- a/src/mon/MonMap.cc +++ b/src/mon/MonMap.cc @@ -372,7 +372,6 @@ void MonMap::dump_summary(Formatter *f) const f->dump_unsigned("num_mons", ranks.size()); } - // an ambiguous mon addr may be legacy or may be msgr2--we aren' sure. // when that happens we need to try them both (unless we can // reasonably infer from the port number which it is). @@ -461,9 +460,29 @@ void MonMap::_add_ambiguous_addr(const string& name, } } +int +MonMap::init_with_addrs(const std::vector& addrs, + bool for_mkfs, + std::string_view prefix) +{ + char id = 'a'; + for (auto& addr : addrs) { + string name{prefix}; + name += id++; + if (addr.v.size() == 1) { + _add_ambiguous_addr(name, addr.front(), 0, 0, for_mkfs); + } else { + // they specified an addrvec, so let's assume they also specified + // the addr *type* and *port*. (we could possibly improve this?) + add(name, addr, 0); + } + } + return 0; +} + int MonMap::init_with_ips(const std::string& ips, bool for_mkfs, - const std::string &prefix) + std::string_view prefix) { vector addrs; if (!parse_ip_port_vec( @@ -473,27 +492,12 @@ int MonMap::init_with_ips(const std::string& ips, } if (addrs.empty()) return -ENOENT; - for (unsigned i=0; i_conf; + + // cct? + auto addrs = cct->get_mon_addrs(); + if (addrs != nullptr && (addrs->size() > 0)) { + return init_with_addrs(*addrs, for_mkfs, "noname-"); + } + // file? if (const auto monmap = conf.get_val("monmap"); !monmap.empty()) { diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h index a149b44ca8b6..44009663edb2 100644 --- a/src/mon/MonMap.h +++ b/src/mon/MonMap.h @@ -30,7 +30,7 @@ #ifdef WITH_SEASTAR -namespace ceph::common { +namespace crimson::common { class ConfigProxy; } #endif @@ -456,6 +456,18 @@ public: static void generate_test_instances(std::list& o); protected: + /** + * build a monmap from a list of entity_addrvec_t's + * + * Give mons dummy names. + * + * @param addrs list of entity_addrvec_t's + * @param prefix prefix to prepend to generated mon names + * @return 0 for success, -errno on error + */ + int init_with_addrs(const std::vector& addrs, + bool for_mkfs, + std::string_view prefix); /** * build a monmap from a list of ips * @@ -467,7 +479,7 @@ protected: */ int init_with_ips(const std::string& ips, bool for_mkfs, - const std::string &prefix); + std::string_view prefix); /** * build a monmap from a list of hostnames * @@ -479,7 +491,7 @@ protected: */ int init_with_hosts(const std::string& hostlist, bool for_mkfs, - const std::string& prefix); + std::string_view prefix); int init_with_config_file(const ConfigProxy& conf, std::ostream& errout); #if WITH_SEASTAR seastar::future<> read_monmap(const std::string& monmap); diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt index f093324e7941..4ede0ecaa954 100644 --- a/src/test/libcephfs/CMakeLists.txt +++ b/src/test/libcephfs/CMakeLists.txt @@ -9,6 +9,7 @@ if(${WITH_CEPHFS}) acl.cc main.cc deleg.cc + monconfig.cc ) target_link_libraries(ceph_test_libcephfs ceph-common diff --git a/src/test/libcephfs/monconfig.cc b/src/test/libcephfs/monconfig.cc new file mode 100644 index 000000000000..c962c4f3ab02 --- /dev/null +++ b/src/test/libcephfs/monconfig.cc @@ -0,0 +1,99 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gtest/gtest.h" +#include "include/cephfs/libcephfs.h" +#include "common/ceph_context.h" +#include +#include +#include +#include +#include + +class MonConfig : public ::testing::Test +{ + protected: + struct ceph_mount_info *ca; + + void SetUp() override { + ASSERT_EQ(0, ceph_create(&ca, NULL)); + ASSERT_EQ(0, ceph_conf_read_file(ca, NULL)); + ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL)); + } + + void TearDown() override { + ceph_shutdown(ca); + } + + // Helper to remove/unset all possible mon information from ConfigProxy + void clear_mon_config(CephContext *cct) { + auto& conf = cct->_conf; + // Clear safe_to_start_threads, allowing updates to config values + conf._clear_safe_to_start_threads(); + ASSERT_EQ(0, conf.set_val("monmap", "", nullptr)); + ASSERT_EQ(0, conf.set_val("mon_host", "", nullptr)); + ASSERT_EQ(0, conf.set_val("mon_dns_srv_name", "", nullptr)); + conf.set_safe_to_start_threads(); + } + + // Helper to test basic operation on a mount + void use_mount(struct ceph_mount_info *mnt, string name_prefix) { + char name[20]; + snprintf(name, sizeof(name), "%s.%d", name_prefix.c_str(), getpid()); + int fd = ceph_open(mnt, name, O_CREAT|O_RDWR, 0644); + ASSERT_LE(0, fd); + + ceph_close(mnt, fd); + } +}; + +TEST_F(MonConfig, MonAddrsMissing) { + CephContext *cct; + + // Test mount failure when there is no known mon config source + cct = ceph_get_mount_context(ca); + ASSERT_NE(nullptr, cct); + clear_mon_config(cct); + + ASSERT_EQ(-ENOENT, ceph_mount(ca, NULL)); +} + +TEST_F(MonConfig, MonAddrsInConfigProxy) { + // Test a successful mount with default mon config source in ConfigProxy + ASSERT_EQ(0, ceph_mount(ca, NULL)); + + use_mount(ca, "foo"); +} + +TEST_F(MonConfig, MonAddrsInCct) { + struct ceph_mount_info *cb; + CephContext *cct; + + // Perform mount to bootstrap mon addrs in CephContext + ASSERT_EQ(0, ceph_mount(ca, NULL)); + + // Reuse bootstrapped CephContext, clearing ConfigProxy mon addr sources + cct = ceph_get_mount_context(ca); + ASSERT_NE(nullptr, cct); + clear_mon_config(cct); + ASSERT_EQ(0, ceph_create_with_context(&cb, cct)); + + // Test a successful mount with only mon values in CephContext + ASSERT_EQ(0, ceph_mount(cb, NULL)); + + use_mount(ca, "bar"); + use_mount(cb, "bar"); + + ceph_shutdown(cb); +} -- 2.47.3