From: Kefu Chai Date: Sat, 11 Aug 2018 03:31:50 +0000 (+0800) Subject: crimson: add MonMap X-Git-Tag: v14.0.1~23^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b980f9f10a5677c6f333a1718b7efe4abe2c5054;p=ceph.git crimson: add MonMap Signed-off-by: Kefu Chai --- diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt index ad72b61dc90..eef0d736c63 100644 --- a/src/crimson/CMakeLists.txt +++ b/src/crimson/CMakeLists.txt @@ -108,6 +108,9 @@ target_link_libraries(crimson-common Boost::random ${NSS_LIBRARIES} ${NSPR_LIBRARIES} ${OPENSSL_LIBRARIES}) +set(crimson_mon_srcs + ${PROJECT_SOURCE_DIR}/src/mon/MonMap.cc + ${PROJECT_SOURCE_DIR}/src/mon/MonSub.cc) set(crimson_net_srcs net/Dispatcher.cc net/Errors.cc @@ -117,6 +120,7 @@ set(crimson_thread_srcs thread/ThreadPool.cc thread/Throttle.cc) add_library(crimson STATIC + ${crimson_mon_srcs} ${crimson_net_srcs} ${crimson_thread_srcs} ${CMAKE_SOURCE_DIR}/src/common/buffer_seastar.cc) diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc index 96854582d92..006d2d49c56 100644 --- a/src/mon/MonMap.cc +++ b/src/mon/MonMap.cc @@ -7,6 +7,13 @@ #include #include +#ifdef WITH_SEASTAR +#include +#include +#include +#include "crimson/common/config_proxy.h" +#endif + #include "common/Formatter.h" #include "include/ceph_features.h" @@ -423,21 +430,6 @@ void MonMap::set_initial_members(CephContext *cct, calc_legacy_ranks(); } -int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout) -{ - int r; - try { - r = read(monmap.c_str()); - } catch (buffer::error&) { - r = -EINVAL; - } - if (r >= 0) - return 0; - errout << "unable to read/decode monmap from " << monmap - << ": " << cpp_strerror(-r) << std::endl; - return r; -} - int MonMap::init_with_config_file(const ConfigProxy& conf, std::ostream& errout) { @@ -502,6 +494,134 @@ int MonMap::init_with_config_file(const ConfigProxy& conf, return 0; } +#ifdef WITH_SEASTAR + +using namespace seastar; + +future<> MonMap::read_monmap(const std::string& monmap) +{ + return open_file_dma(monmap, open_flags::ro).then([this] (file f) { + return f.size().then([this, f = std::move(f)](size_t s) { + return do_with(make_file_input_stream(f), [this, s](input_stream& in) { + return in.read_exactly(s).then([this](temporary_buffer buf) { + bufferlist bl; + bl.append(buffer::create(std::move(buf))); + decode(bl); + }); + }); + }); + }); +} + +future<> MonMap::init_with_dns_srv(const std::string& name) +{ + string domain; + string service = name; + // check if domain is also provided and extract it from srv_name + size_t idx = name.find("_"); + if (idx != name.npos) { + domain = name.substr(idx + 1); + service = name.substr(0, idx); + } + return net::dns::get_srv_records( + net::dns_resolver::srv_proto::tcp, + service, domain).then([this](net::dns_resolver::srv_records records) { + return parallel_for_each(records, [this](auto record) { + return net::dns::resolve_name(record.target).then( + [record,this](net::inet_address a) { + // the resolved address does not contain ceph specific info like nonce + // nonce or msgr proto (legacy, msgr2), so set entity_addr_t manually + entity_addr_t addr; + addr.set_type(entity_addr_t::TYPE_LEGACY); + addr.set_family(int(a.in_family())); + addr.set_port(record.port); + switch (a.in_family()) { + case net::inet_address::family::INET: + addr.in4_addr().sin_addr = a; + break; + case net::inet_address::family::INET6: + addr.in6_addr().sin6_addr = a; + break; + } + add(mon_info_t{record.target, addr, record.priority}); + }); + }); + }).handle_exception_type([](const std::system_error& e) { + // ignore DNS failures + return seastar::make_ready_future<>(); + }); +} + +seastar::future<> MonMap::build_monmap(const ceph::common::ConfigProxy& conf) +{ + // -m foo? + if (const auto mon_host = conf.get_val("mon_host"); + !mon_host.empty()) { + if (auto ret = init_with_ips(mon_host, "noname-"); ret == 0) { + return make_ready_future<>(); + } + // TODO: resolve_addrs() is a blocking call + if (auto ret = init_with_hosts(mon_host, "noname-"); ret == 0) { + return make_ready_future<>(); + } else { + throw std::runtime_error(cpp_strerror(ret)); + } + } + + // What monitors are in the config file? + ostringstream errout; + if (auto ret = init_with_config_file(conf, errout); ret < 0) { + throw std::runtime_error(errout.str()); + } + if (size() > 0) { + return make_ready_future<>(); + } + // no info found from conf options lets try use DNS SRV records + const string srv_name = conf.get_val("mon_dns_srv_name"); + return init_with_dns_srv(srv_name).then([this] { + if (size() == 0) { + throw std::runtime_error("no monitors specified to connect to."); + } + }); +} + +future<> MonMap::build_initial(const ceph::common::ConfigProxy& conf) +{ + // file? + if (const auto monmap = conf.get_val("monmap"); + !monmap.empty()) { + return read_monmap(monmap); + } else { + // fsid from conf? + if (const auto new_fsid = conf.get_val("fsid"); + !new_fsid.is_zero()) { + fsid = new_fsid; + } + return build_monmap(conf).then([this] { + created = ceph_clock_now(); + last_changed = created; + calc_legacy_ranks(); + }); + } +} + +#else // WITH_SEASTAR + +int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout) +{ + int r; + try { + r = read(monmap.c_str()); + } catch (buffer::error&) { + r = -EINVAL; + } + if (r >= 0) + return 0; + errout << "unable to read/decode monmap from " << monmap + << ": " << cpp_strerror(-r) << std::endl; + return r; +} + int MonMap::init_with_dns_srv(CephContext* cct, std::string srv_name, std::ostream& errout) @@ -580,3 +700,4 @@ int MonMap::build_initial(CephContext *cct, ostream& errout) calc_legacy_ranks(); return 0; } +#endif // WITH_SEASTAR diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h index 4eeea332101..ff74f194510 100644 --- a/src/mon/MonMap.h +++ b/src/mon/MonMap.h @@ -15,6 +15,9 @@ #ifndef CEPH_MONMAP_H #define CEPH_MONMAP_H +#ifdef WITH_SEASTAR +#include +#endif #include "common/config_fwd.h" @@ -25,6 +28,12 @@ #include "msg/Message.h" +#ifdef WITH_SEASTAR +namespace ceph::common { + class ConfigProxy; +} +#endif + namespace ceph { class Formatter; } @@ -343,8 +352,11 @@ public: * @param cct context (and associated config) * @param errout ostream to send error messages too */ +#ifdef WITH_SEASTAR + seastar::future<> build_initial(const ceph::common::ConfigProxy& conf); +#else int build_initial(CephContext *cct, ostream& errout); - +#endif /** * filter monmap given a set of initial members. * @@ -368,8 +380,7 @@ public: void dump(ceph::Formatter *f) const; static void generate_test_instances(list& o); -private: - int init_with_monmap(const std::string& monmap, std::ostream& errout); +protected: /** * build a monmap from a list of ips * @@ -392,10 +403,20 @@ private: */ int init_with_hosts(const std::string& hostlist, const std::string& prefix); - int init_with_mon_host(const std::string& mon_host, std::ostream& errout); int init_with_config_file(const ConfigProxy& conf, std::ostream& errout); +#if WITH_SEASTAR + seastar::future<> read_monmap(const std::string& monmap); + /// try to build monmap with different settings, like + /// mon_host, mon* sections, and mon_dns_srv_name + seastar::future<> build_monmap(const ceph::common::ConfigProxy& conf); + /// initialize monmap by resolving given service name + seastar::future<> init_with_dns_srv(const std::string& name); +#else + /// read from encoded monmap file + int init_with_monmap(const std::string& monmap, std::ostream& errout); int init_with_dns_srv(CephContext* cct, std::string srv_name, std::ostream& errout); +#endif }; WRITE_CLASS_ENCODER_FEATURES(MonMap)