crimson: add MonMap

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt
index ad72b61dc90ce9e3cae5530f62e2f114677b4766..eef0d736c63ff86bcf1bbae9894b54fa8b588f7f 100644 (file)
--- a/src/crimson/CMakeLists.txt
+++ b/src/crimson/CMakeLists.txt
@@ -108,6 +108,9 @@ target_link_libraries(crimson-common
     Boost::random
     ${NSS_LIBRARIES} ${NSPR_LIBRARIES} ${OPENSSL_LIBRARIES})
 
+set(crimson_mon_srcs
+  ${PROJECT_SOURCE_DIR}/src/mon/MonMap.cc
+  ${PROJECT_SOURCE_DIR}/src/mon/MonSub.cc)
 set(crimson_net_srcs
   net/Dispatcher.cc
   net/Errors.cc
@@ -117,6 +120,7 @@ set(crimson_thread_srcs
   thread/ThreadPool.cc
   thread/Throttle.cc)
 add_library(crimson STATIC
+  ${crimson_mon_srcs}
   ${crimson_net_srcs}
   ${crimson_thread_srcs}
   ${CMAKE_SOURCE_DIR}/src/common/buffer_seastar.cc)

diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
index 96854582d92974d12a7efc06841098814c468030..006d2d49c5690ef11f49336bcefea08001e8007e 100644 (file)
--- a/src/mon/MonMap.cc
+++ b/src/mon/MonMap.cc
@@ -7,6 +7,13 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#ifdef WITH_SEASTAR
+#include <seastar/core/fstream.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/net/dns.hh>
+#include "crimson/common/config_proxy.h"
+#endif
+
 #include "common/Formatter.h"
 
 #include "include/ceph_features.h"
@@ -423,21 +430,6 @@ void MonMap::set_initial_members(CephContext *cct,
   calc_legacy_ranks();
 }
 
-int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout)
-{
-  int r;
-  try {
-    r = read(monmap.c_str());
-  } catch (buffer::error&) {
-    r = -EINVAL;
-  }
-  if (r >= 0)
-    return 0;
-  errout << "unable to read/decode monmap from " << monmap
-         << ": " << cpp_strerror(-r) << std::endl;
-  return r;
-}
-
 int MonMap::init_with_config_file(const ConfigProxy& conf,
                                   std::ostream& errout)
 {
@@ -502,6 +494,134 @@ int MonMap::init_with_config_file(const ConfigProxy& conf,
   return 0;
 }
 
+#ifdef WITH_SEASTAR
+
+using namespace seastar;
+
+future<> MonMap::read_monmap(const std::string& monmap)
+{
+  return open_file_dma(monmap, open_flags::ro).then([this] (file f) {
+    return f.size().then([this, f = std::move(f)](size_t s) {
+      return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
+        return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
+          bufferlist bl;
+          bl.append(buffer::create(std::move(buf)));
+          decode(bl);
+        });
+      });
+    });
+  });
+}
+
+future<> MonMap::init_with_dns_srv(const std::string& name)
+{
+  string domain;
+  string service = name;
+  // check if domain is also provided and extract it from srv_name
+  size_t idx = name.find("_");
+  if (idx != name.npos) {
+    domain = name.substr(idx + 1);
+    service = name.substr(0, idx);
+  }
+  return net::dns::get_srv_records(
+      net::dns_resolver::srv_proto::tcp,
+      service, domain).then([this](net::dns_resolver::srv_records records) {
+    return parallel_for_each(records, [this](auto record) {
+      return net::dns::resolve_name(record.target).then(
+          [record,this](net::inet_address a) {
+       // the resolved address does not contain ceph specific info like nonce
+       // nonce or msgr proto (legacy, msgr2), so set entity_addr_t manually
+       entity_addr_t addr;
+       addr.set_type(entity_addr_t::TYPE_LEGACY);
+       addr.set_family(int(a.in_family()));
+       addr.set_port(record.port);
+       switch (a.in_family()) {
+       case net::inet_address::family::INET:
+         addr.in4_addr().sin_addr = a;
+         break;
+       case net::inet_address::family::INET6:
+         addr.in6_addr().sin6_addr = a;
+         break;
+       }
+       add(mon_info_t{record.target, addr, record.priority});
+      });
+    });
+  }).handle_exception_type([](const std::system_error& e) {
+    // ignore DNS failures
+    return seastar::make_ready_future<>();
+  });
+}
+
+seastar::future<> MonMap::build_monmap(const ceph::common::ConfigProxy& conf)
+{
+  // -m foo?
+  if (const auto mon_host = conf.get_val<std::string>("mon_host");
+      !mon_host.empty()) {
+    if (auto ret = init_with_ips(mon_host, "noname-"); ret == 0) {
+      return make_ready_future<>();
+    }
+    // TODO: resolve_addrs() is a blocking call
+    if (auto ret = init_with_hosts(mon_host, "noname-"); ret == 0) {
+      return make_ready_future<>();
+    } else {
+      throw std::runtime_error(cpp_strerror(ret));
+    }
+  }
+
+  // What monitors are in the config file?
+  ostringstream errout;
+  if (auto ret = init_with_config_file(conf, errout); ret < 0) {
+    throw std::runtime_error(errout.str());
+  }
+  if (size() > 0) {
+    return make_ready_future<>();
+  }
+  // no info found from conf options lets try use DNS SRV records
+  const string srv_name = conf.get_val<std::string>("mon_dns_srv_name");
+  return init_with_dns_srv(srv_name).then([this] {
+    if (size() == 0) {
+      throw std::runtime_error("no monitors specified to connect to.");
+    }
+  });
+}
+
+future<> MonMap::build_initial(const ceph::common::ConfigProxy& conf)
+{
+  // file?
+  if (const auto monmap = conf.get_val<std::string>("monmap");
+      !monmap.empty()) {
+    return read_monmap(monmap);
+  } else {
+    // fsid from conf?
+    if (const auto new_fsid = conf.get_val<uuid_d>("fsid");
+        !new_fsid.is_zero()) {
+      fsid = new_fsid;
+    }
+    return build_monmap(conf).then([this] {
+      created = ceph_clock_now();
+      last_changed = created;
+      calc_legacy_ranks();
+    });
+  }
+}
+
+#else  // WITH_SEASTAR
+
+int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout)
+{
+  int r;
+  try {
+    r = read(monmap.c_str());
+  } catch (buffer::error&) {
+    r = -EINVAL;
+  }
+  if (r >= 0)
+    return 0;
+  errout << "unable to read/decode monmap from " << monmap
+         << ": " << cpp_strerror(-r) << std::endl;
+  return r;
+}
+
 int MonMap::init_with_dns_srv(CephContext* cct,
                               std::string srv_name,
                               std::ostream& errout)
@@ -580,3 +700,4 @@ int MonMap::build_initial(CephContext *cct, ostream& errout)
   calc_legacy_ranks();
   return 0;
 }
+#endif // WITH_SEASTAR

diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h
index 4eeea332101725d2c508505b1e91de10c06a47c5..ff74f194510c10c238b6947171000365c1d913cc 100644 (file)
--- a/src/mon/MonMap.h
+++ b/src/mon/MonMap.h
@@ -15,6 +15,9 @@
 #ifndef CEPH_MONMAP_H
 #define CEPH_MONMAP_H
 
+#ifdef WITH_SEASTAR
+#include <seastar/core/future.hh>
+#endif
 
 #include "common/config_fwd.h"
 
@@ -25,6 +28,12 @@
 #include "msg/Message.h"
 
 
+#ifdef WITH_SEASTAR
+namespace ceph::common {
+  class ConfigProxy;
+}
+#endif
+
 namespace ceph {
   class Formatter;
 }
@@ -343,8 +352,11 @@ public:
    * @param cct context (and associated config)
    * @param errout ostream to send error messages too
    */
+#ifdef WITH_SEASTAR
+  seastar::future<> build_initial(const ceph::common::ConfigProxy& conf);
+#else
   int build_initial(CephContext *cct, ostream& errout);
-
+#endif
   /**
    * filter monmap given a set of initial members.
    *
@@ -368,8 +380,7 @@ public:
   void dump(ceph::Formatter *f) const;
 
   static void generate_test_instances(list<MonMap*>& o);
-private:
-  int init_with_monmap(const std::string& monmap, std::ostream& errout);
+protected:
   /**
    * build a monmap from a list of ips
    *
@@ -392,10 +403,20 @@ private:
    */
   int init_with_hosts(const std::string& hostlist,
                         const std::string& prefix);
-  int init_with_mon_host(const std::string& mon_host, std::ostream& errout);
   int init_with_config_file(const ConfigProxy& conf, std::ostream& errout);
+#if WITH_SEASTAR
+  seastar::future<> read_monmap(const std::string& monmap);
+  /// try to build monmap with different settings, like
+  /// mon_host, mon* sections, and mon_dns_srv_name
+  seastar::future<> build_monmap(const ceph::common::ConfigProxy& conf);
+  /// initialize monmap by resolving given service name
+  seastar::future<> init_with_dns_srv(const std::string& name);
+#else
+  /// read from encoded monmap file
+  int init_with_monmap(const std::string& monmap, std::ostream& errout);
   int init_with_dns_srv(CephContext* cct, std::string srv_name,
                        std::ostream& errout);
+#endif
 };
 WRITE_CLASS_ENCODER_FEATURES(MonMap)