mon: let users specify a crush location on boot, and send it in MMonJoin

Go to some effort to look at our location in the monmap and update it the
same way we update names or IP addresses when necessary. Let users
pass in the location on the CLI via "--set-crush-location".

Signed-off-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index 71c53d6dbca66810c493dc1235ea79522bfd93c2..a468d8b9d16d29b8b2dc0c13c697a5ddb6921371 100644 (file)
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -210,6 +210,8 @@ static void usage()
        << "        extract the monmap from the local monitor store and exit\n"
        << "  --mon-data <directory>\n"
        << "        where the mon store and keyring are located\n"
+       << "  --set-crush-location <bucket>=<foo>"
+       << "        sets monitor's crush bucket location (only for stretch mode)"
        << std::endl;
   generic_server_usage();
 }
@@ -248,7 +250,7 @@ int main(int argc, const char **argv)
   bool compact = false;
   bool force_sync = false;
   bool yes_really = false;
-  std::string osdmapfn, inject_monmap, extract_monmap;
+  std::string osdmapfn, inject_monmap, extract_monmap, crush_loc;
 
   vector<const char*> args;
   argv_to_vec(argc, argv, args);
@@ -331,6 +333,8 @@ int main(int argc, const char **argv)
       inject_monmap = val;
     } else if (ceph_argparse_witharg(args, i, &val, "--extract-monmap", (char*)NULL)) {
       extract_monmap = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--set-crush-location", (char*)NULL)) {
+      crush_loc = val;
     } else {
       ++i;
     }
@@ -896,6 +900,7 @@ int main(int argc, const char **argv)
   msgr->start();
   mgr_msgr->start();
 
+  mon->set_mon_crush_location(crush_loc);
   mon->init();
 
   register_async_signal_handler_oneshot(SIGINT, handle_mon_signal);

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index acb8eb608c766e9f88f488737310456912fbce69..30b977723662d9a47f4f1c262cf308533779a631 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2104,16 +2104,25 @@ void Monitor::handle_probe_reply(MonOpRequestRef op)
              << " vs my version " << paxos->get_version()
              << " (ok)"
              << dendl;
-
-    if (monmap->contains(name) &&
-        !monmap->get_addrs(name).front().is_blank_ip()) {
+    bool in_map = false;
+    const auto my_info = monmap->mon_info.find(name);
+    const map<string,string> *map_crush_loc{nullptr};
+    if (my_info != monmap->mon_info.end()) {
+      in_map = true;
+      map_crush_loc = &my_info->second.crush_loc;
+    }
+    if (in_map &&
+       !monmap->get_addrs(name).front().is_blank_ip() &&
+       (!need_set_crush_loc || (*map_crush_loc == crush_loc))) {
       // i'm part of the cluster; just initiate a new election
       start_election();
     } else {
-      dout(10) << " ready to join, but i'm not in the monmap or my addr is blank, trying to join" << dendl;
-      send_mon_message(
-       new MMonJoin(monmap->fsid, name, messenger->get_myaddrs()),
-       *m->quorum.begin());
+      dout(10) << " ready to join, but i'm not in the monmap/"
+       "my addr is blank/location is wrong, trying to join" << dendl;
+      send_mon_message(new MMonJoin(monmap->fsid, name,
+                                   messenger->get_myaddrs(), crush_loc,
+                                   need_set_crush_loc),
+                      *m->quorum.begin());
     }
   } else {
     if (monmap->contains(m->name)) {
@@ -2377,13 +2386,18 @@ void Monitor::finish_election()
     authmon()->_set_mon_num_rank(monmap->size(), rank);
   }
 
-  // am i named properly?
+  // am i named and located properly?
   string cur_name = monmap->get_name(messenger->get_myaddrs());
-  if (cur_name != name) {
-    dout(10) << " renaming myself from " << cur_name << " -> " << name << dendl;
-    send_mon_message(
-      new MMonJoin(monmap->fsid, name, messenger->get_myaddrs()),
-      *quorum.begin());
+  const auto my_infop = monmap->mon_info.find(cur_name);
+  const map<string,string>& map_crush_loc = my_infop->second.crush_loc;
+  
+  if (cur_name != name ||
+      (need_set_crush_loc && map_crush_loc != crush_loc)) {
+    dout(10) << " renaming/moving myself from " << cur_name << "/"
+            << map_crush_loc <<" -> " << name << "/" << crush_loc << dendl;
+    send_mon_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddrs(),
+                                 crush_loc, need_set_crush_loc),
+                    *quorum.begin());
     return;
   }
   do_stretch_mode_election_work();
@@ -6485,8 +6499,26 @@ int Monitor::ms_handle_authentication(Connection *con)
   return ret;
 }
 
+void Monitor::set_mon_crush_location(const string& loc)
+{
+  if (loc.empty()) {
+    return;
+  }
+  vector<string> loc_vec;
+  loc_vec.push_back(loc);
+  CrushWrapper::parse_loc_map(loc_vec, &crush_loc);
+  need_set_crush_loc = true;
+}
+
 void Monitor::notify_new_monmap(bool can_change_external_state)
 {
+  if (need_set_crush_loc) {
+    auto my_info_i = monmap->mon_info.find(name);
+    if (my_info_i != monmap->mon_info.end() &&
+       my_info_i->second.crush_loc == crush_loc) {
+      need_set_crush_loc = false;
+    }
+  }
   elector.notify_strategy_maybe_changed(monmap->strategy);
   dout(30) << __func__ << "we have " << monmap->removed_ranks.size() << " removed ranks" << dendl;
   for (auto i = monmap->removed_ranks.rbegin();

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 616272afb7d666a8bbc8aac9dd73488dd99e8422..8fbae34b1ea7fe0d9ad7a7a170441c7697fa5a9c 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -259,6 +259,9 @@ private:
   bool session_stretch_allowed(MonSession *s, MonOpRequestRef& op);
   void disconnect_disallowed_stretch_sessions();
   void set_elector_disallowed_leaders(bool allow_election);
+
+  map <string,string> crush_loc;
+  bool need_set_crush_loc{false};
 public:
   bool is_stretch_mode() { return stretch_mode_engaged; }
   bool is_degraded_stretch_mode() { return degraded_stretch_mode; }
@@ -272,6 +275,7 @@ public:
   void trigger_healthy_stretch_mode();
   void set_healthy_stretch_mode();
   void enable_stretch_mode();
+  void set_mon_crush_location(const string& loc);
 
   
 private: