From c3eb38a36c0c45cef90767e221fcd53eebe27be9 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Mon, 29 Mar 2021 23:06:53 +0000 Subject: [PATCH] mon: let users specify a crush location on boot, and send it in MMonJoin Go to some effort to look at our location in the monmap and update it the same way we update names or IP addresses when necessary. Let users pass in the location on the CLI via "--set-crush-location". Signed-off-by: Greg Farnum (cherry picked from commit 48886aa12a351919d8165d449c2fb72a896fc589) --- src/ceph_mon.cc | 7 +++++- src/mon/Monitor.cc | 58 +++++++++++++++++++++++++++++++++++----------- src/mon/Monitor.h | 4 ++++ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 71c53d6dbca66..a468d8b9d16d2 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -210,6 +210,8 @@ static void usage() << " extract the monmap from the local monitor store and exit\n" << " --mon-data \n" << " where the mon store and keyring are located\n" + << " --set-crush-location =" + << " sets monitor's crush bucket location (only for stretch mode)" << std::endl; generic_server_usage(); } @@ -248,7 +250,7 @@ int main(int argc, const char **argv) bool compact = false; bool force_sync = false; bool yes_really = false; - std::string osdmapfn, inject_monmap, extract_monmap; + std::string osdmapfn, inject_monmap, extract_monmap, crush_loc; vector args; argv_to_vec(argc, argv, args); @@ -331,6 +333,8 @@ int main(int argc, const char **argv) inject_monmap = val; } else if (ceph_argparse_witharg(args, i, &val, "--extract-monmap", (char*)NULL)) { extract_monmap = val; + } else if (ceph_argparse_witharg(args, i, &val, "--set-crush-location", (char*)NULL)) { + crush_loc = val; } else { ++i; } @@ -896,6 +900,7 @@ int main(int argc, const char **argv) msgr->start(); mgr_msgr->start(); + mon->set_mon_crush_location(crush_loc); mon->init(); register_async_signal_handler_oneshot(SIGINT, handle_mon_signal); diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index bc34069f39d8c..5f0584bb14fec 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2105,16 +2105,25 @@ void Monitor::handle_probe_reply(MonOpRequestRef op) << " vs my version " << paxos->get_version() << " (ok)" << dendl; - - if (monmap->contains(name) && - !monmap->get_addrs(name).front().is_blank_ip()) { + bool in_map = false; + const auto my_info = monmap->mon_info.find(name); + const map *map_crush_loc{nullptr}; + if (my_info != monmap->mon_info.end()) { + in_map = true; + map_crush_loc = &my_info->second.crush_loc; + } + if (in_map && + !monmap->get_addrs(name).front().is_blank_ip() && + (!need_set_crush_loc || (*map_crush_loc == crush_loc))) { // i'm part of the cluster; just initiate a new election start_election(); } else { - dout(10) << " ready to join, but i'm not in the monmap or my addr is blank, trying to join" << dendl; - send_mon_message( - new MMonJoin(monmap->fsid, name, messenger->get_myaddrs()), - *m->quorum.begin()); + dout(10) << " ready to join, but i'm not in the monmap/" + "my addr is blank/location is wrong, trying to join" << dendl; + send_mon_message(new MMonJoin(monmap->fsid, name, + messenger->get_myaddrs(), crush_loc, + need_set_crush_loc), + *m->quorum.begin()); } } else { if (monmap->contains(m->name)) { @@ -2378,13 +2387,18 @@ void Monitor::finish_election() authmon()->_set_mon_num_rank(monmap->size(), rank); } - // am i named properly? + // am i named and located properly? string cur_name = monmap->get_name(messenger->get_myaddrs()); - if (cur_name != name) { - dout(10) << " renaming myself from " << cur_name << " -> " << name << dendl; - send_mon_message( - new MMonJoin(monmap->fsid, name, messenger->get_myaddrs()), - *quorum.begin()); + const auto my_infop = monmap->mon_info.find(cur_name); + const map& map_crush_loc = my_infop->second.crush_loc; + + if (cur_name != name || + (need_set_crush_loc && map_crush_loc != crush_loc)) { + dout(10) << " renaming/moving myself from " << cur_name << "/" + << map_crush_loc <<" -> " << name << "/" << crush_loc << dendl; + send_mon_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddrs(), + crush_loc, need_set_crush_loc), + *quorum.begin()); return; } do_stretch_mode_election_work(); @@ -6505,8 +6519,26 @@ int Monitor::ms_handle_authentication(Connection *con) return ret; } +void Monitor::set_mon_crush_location(const string& loc) +{ + if (loc.empty()) { + return; + } + vector loc_vec; + loc_vec.push_back(loc); + CrushWrapper::parse_loc_map(loc_vec, &crush_loc); + need_set_crush_loc = true; +} + void Monitor::notify_new_monmap(bool can_change_external_state) { + if (need_set_crush_loc) { + auto my_info_i = monmap->mon_info.find(name); + if (my_info_i != monmap->mon_info.end() && + my_info_i->second.crush_loc == crush_loc) { + need_set_crush_loc = false; + } + } elector.notify_strategy_maybe_changed(monmap->strategy); dout(30) << __func__ << "we have " << monmap->removed_ranks.size() << " removed ranks" << dendl; for (auto i = monmap->removed_ranks.rbegin(); diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index cd624f001f6d9..d5751cc9d366e 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -259,6 +259,9 @@ private: bool session_stretch_allowed(MonSession *s, MonOpRequestRef& op); void disconnect_disallowed_stretch_sessions(); void set_elector_disallowed_leaders(bool allow_election); + + map crush_loc; + bool need_set_crush_loc{false}; public: bool is_stretch_mode() { return stretch_mode_engaged; } bool is_degraded_stretch_mode() { return degraded_stretch_mode; } @@ -272,6 +275,7 @@ public: void trigger_healthy_stretch_mode(); void set_healthy_stretch_mode(); void enable_stretch_mode(); + void set_mon_crush_location(const string& loc); private: -- 2.39.5