From 3d34c4ebd2f8a0dcd88a26c59ee3a9206d1cd9bb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 7 Jan 2019 12:52:43 -0600 Subject: [PATCH] mon: adjust mon sync, extra_probe_peers to use addrvec The peer addr stuff via asok is a bit fragile because the user must provide an exact addrvec matching the mon to avoid some weirdness, but it's rarely used, and the fix would be some robustness/tolerance in the messenger that is a bigger project than this. Signed-off-by: Sage Weil --- doc/dev/mon-bootstrap.rst | 19 +++++- src/mon/MonMap.cc | 6 +- src/mon/MonMap.h | 2 +- src/mon/Monitor.cc | 122 +++++++++++++++++++++++--------------- src/mon/Monitor.h | 8 +-- src/tools/monmaptool.cc | 2 +- 6 files changed, 99 insertions(+), 60 deletions(-) diff --git a/doc/dev/mon-bootstrap.rst b/doc/dev/mon-bootstrap.rst index eb99a477d867..2b67b470765c 100644 --- a/doc/dev/mon-bootstrap.rst +++ b/doc/dev/mon-bootstrap.rst @@ -172,12 +172,25 @@ example:: ceph-mon --mkfs -i --fsid --keyring --public-addr -Once the daemon starts, you can give it one or more peer addresses to join with:: +Once the daemon starts, you can give it one or more peer addresses (preferably a bare IP address with no port; the mon will set the addr types and ports for you) to join with:: ceph daemon mon. add_bootstrap_peer_hint -This monitor will never participate in cluster creation; it can only join an existing -cluster. +Alternatively, you can explicitly specify the addrvec_t with:: + + ceph daemon mon. add_bootstrap_peer_hintv + +For example,:: + + ceph daemon mon.new add_bootstrap_peer_hintv v2:1.2.3.4:3300,v1:1.2.3.4:6789 + +This monitor will never participate in cluster creation; it can only +join an existing cluster. + +Note that the address(es) specified should match exactly the addresses +the new monitor is binding too. If, for example, the new mon binds to +only a v2 address but a v2 and v1 address are provided, there is some +possibility of confusion in the mons. Expanding with initial members ------------------------------ diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc index 53bc8fc4953f..372d0161b4a9 100644 --- a/src/mon/MonMap.cc +++ b/src/mon/MonMap.cc @@ -468,7 +468,7 @@ void MonMap::set_initial_members(CephContext *cct, list& initial_members, string my_name, const entity_addrvec_t& my_addrs, - set *removed) + set *removed) { // remove non-initial members unsigned i = 0; @@ -484,9 +484,7 @@ void MonMap::set_initial_members(CephContext *cct, lgeneric_dout(cct, 1) << " removing " << get_name(i) << " " << get_addrs(i) << dendl; if (removed) { - for (auto& j : get_addrs(i).v) { - removed->insert(j); - } + removed->insert(get_addrs(i)); } remove(n); ceph_assert(!contains(n)); diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h index 7ea959e67377..62f313e7b957 100644 --- a/src/mon/MonMap.h +++ b/src/mon/MonMap.h @@ -403,7 +403,7 @@ public: list& initial_members, string my_name, const entity_addrvec_t& my_addrs, - set *removed); + set *removed); void print(ostream& out) const; void print_summary(ostream& out) const; diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index eb839bcaf81b..e3dcd96b1bce 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -324,7 +324,8 @@ void Monitor::do_admin_command(std::string_view command, const cmdmap_t& cmdmap, goto abort; } sync_force(f.get(), ss); - } else if (command.compare(0, 23, "add_bootstrap_peer_hint") == 0) { + } else if (command.compare(0, 23, "add_bootstrap_peer_hint") == 0 || + command.compare(0, 24, "add_bootstrap_peer_hintv") == 0) { if (!_add_bootstrap_peer_hint(command, cmdmap, ss)) goto abort; } else if (command == "quorum enter") { @@ -809,6 +810,13 @@ int Monitor::preinit() "add peer address as potential bootstrap" " peer for cluster bringup"); ceph_assert(r == 0); + r = admin_socket->register_command("add_bootstrap_peer_hintv", + "add_bootstrap_peer_hintv name=addrv," + "type=CephString", + admin_hook, + "add peer address vector as potential bootstrap" + " peer for cluster bringup"); + ceph_assert(r == 0); r = admin_socket->register_command("quorum enter", "quorum enter", admin_hook, "force monitor back into quorum"); @@ -1086,16 +1094,11 @@ void Monitor::bootstrap() new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined), i); } - for (set::iterator p = extra_probe_peers.begin(); - p != extra_probe_peers.end(); - ++p) { - if (*p != messenger->get_myaddr()) { - entity_inst_t i; - i.name = entity_name_t::MON(-1); - i.addr = *p; - messenger->send_message( + for (auto& av : extra_probe_peers) { + if (av != messenger->get_myaddrs()) { + messenger->send_to_mon( new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined), - i); + av); } } } @@ -1104,32 +1107,55 @@ bool Monitor::_add_bootstrap_peer_hint(std::string_view cmd, const cmdmap_t& cmdmap, ostream& ss) { - string addrstr; - if (!cmd_getval(g_ceph_context, cmdmap, "addr", addrstr)) { - ss << "unable to parse address string value '" - << cmd_vartype_stringify(cmdmap.at("addr")) << "'"; - return false; - } - dout(10) << "_add_bootstrap_peer_hint '" << cmd << "' '" - << addrstr << "'" << dendl; - - entity_addr_t addr; - const char *end = 0; - if (!addr.parse(addrstr.c_str(), &end)) { - ss << "failed to parse addr '" << addrstr << "'; syntax is 'add_bootstrap_peer_hint ip[:port]'"; - return false; - } - if (is_leader() || is_peon()) { ss << "mon already active; ignoring bootstrap hint"; return true; } - if (addr.get_port() == 0) - addr.set_port(CEPH_MON_PORT_LEGACY); + entity_addrvec_t addrs; + string addrstr; + if (cmd_getval(g_ceph_context, cmdmap, "addr", addrstr)) { + dout(10) << "_add_bootstrap_peer_hint '" << cmd << "' addr '" + << addrstr << "'" << dendl; + + entity_addr_t addr; + const char *end = 0; + if (!addr.parse(addrstr.c_str(), &end, entity_addr_t::TYPE_ANY)) { + ss << "failed to parse addrs '" << addrstr + << "'; syntax is 'add_bootstrap_peer_hint ip[:port]'"; + return false; + } - extra_probe_peers.insert(addr); - ss << "adding peer " << addr << " to list: " << extra_probe_peers; + addrs.v.push_back(addr); + if (addr.get_port() == 0) { + addrs.v[0].set_type(entity_addr_t::TYPE_MSGR2); + addrs.v[0].set_port(CEPH_MON_PORT_IANA); + addrs.v.push_back(addr); + addrs.v[1].set_type(entity_addr_t::TYPE_LEGACY); + addrs.v[1].set_port(CEPH_MON_PORT_LEGACY); + } else if (addr.get_type() == entity_addr_t::TYPE_ANY) { + if (addr.get_port() == CEPH_MON_PORT_LEGACY) { + addrs.v[0].set_type(entity_addr_t::TYPE_LEGACY); + } else { + addrs.v[0].set_type(entity_addr_t::TYPE_MSGR2); + } + } + } else if (cmd_getval(g_ceph_context, cmdmap, "addrv", addrstr)) { + dout(10) << "_add_bootstrap_peer_hintv '" << cmd << "' addrv '" + << addrstr << "'" << dendl; + const char *end = 0; + if (!addrs.parse(addrstr.c_str(), &end)) { + ss << "failed to parse addrs '" << addrstr + << "'; syntax is 'add_bootstrap_peer_hintv v2:ip:port[,v1:ip:port]'"; + return false; + } + } else { + ss << "no addr or addrv provided"; + return false; + } + + extra_probe_peers.insert(addrs); + ss << "adding peer " << addrs << " to list: " << extra_probe_peers; return true; } @@ -1245,7 +1271,7 @@ void Monitor::sync_reset_requester() sync_timeout_event = NULL; } - sync_provider = entity_inst_t(); + sync_provider = entity_addrvec_t(); sync_cookie = 0; sync_full = false; sync_start_version = 0; @@ -1257,9 +1283,9 @@ void Monitor::sync_reset_provider() sync_providers.clear(); } -void Monitor::sync_start(entity_inst_t &other, bool full) +void Monitor::sync_start(entity_addrvec_t &addrs, bool full) { - dout(10) << __func__ << " " << other << (full ? " full" : " recent") << dendl; + dout(10) << __func__ << " " << addrs << (full ? " full" : " recent") << dendl; ceph_assert(state == STATE_PROBING || state == STATE_SYNCHRONIZING); @@ -1300,14 +1326,14 @@ void Monitor::sync_start(entity_inst_t &other, bool full) // assume 'other' as the leader. We will update the leader once we receive // a reply to the sync start. - sync_provider = other; + sync_provider = addrs; sync_reset_timeout(); MMonSync *m = new MMonSync(sync_full ? MMonSync::OP_GET_COOKIE_FULL : MMonSync::OP_GET_COOKIE_RECENT); if (!sync_full) m->last_committed = paxos->get_version(); - messenger->send_message(m, sync_provider); + messenger->send_to_mon(m, sync_provider); } void Monitor::sync_stash_critical_state(MonitorDBStore::TransactionRef t) @@ -1446,7 +1472,7 @@ void Monitor::handle_sync_get_cookie(MonOpRequestRef op) SyncProvider& sp = sync_providers[cookie]; sp.cookie = cookie; - sp.entity = m->get_source_inst(); + sp.addrs = m->get_source_addrs(); sp.reset_timeout(g_ceph_context, g_conf()->mon_sync_timeout * 2); set sync_targets; @@ -1547,7 +1573,7 @@ void Monitor::handle_sync_cookie(MonOpRequestRef op) dout(10) << __func__ << " already have a cookie, ignoring" << dendl; return; } - if (m->get_source_inst() != sync_provider) { + if (m->get_source_addrs() != sync_provider) { dout(10) << __func__ << " source does not match, discarding" << dendl; return; } @@ -1568,7 +1594,7 @@ void Monitor::sync_get_next_chunk() usleep((long long)(g_conf()->mon_inject_sync_get_chunk_delay * 1000000.0)); } MMonSync *r = new MMonSync(MMonSync::OP_GET_CHUNK, sync_cookie); - messenger->send_message(r, sync_provider); + messenger->send_to_mon(r, sync_provider); ceph_assert(g_conf()->mon_sync_requester_kill_at != 4); } @@ -1582,7 +1608,7 @@ void Monitor::handle_sync_chunk(MonOpRequestRef op) dout(10) << __func__ << " cookie does not match, discarding" << dendl; return; } - if (m->get_source_inst() != sync_provider) { + if (m->get_source_addrs() != sync_provider) { dout(10) << __func__ << " source does not match, discarding" << dendl; return; } @@ -1642,7 +1668,8 @@ void Monitor::sync_trim_providers() map::iterator p = sync_providers.begin(); while (p != sync_providers.end()) { if (now > p->second.timeout) { - dout(10) << __func__ << " expiring cookie " << p->second.cookie << " for " << p->second.entity << dendl; + dout(10) << __func__ << " expiring cookie " << p->second.cookie + << " for " << p->second.addrs << dendl; sync_providers.erase(p++); } else { ++p; @@ -1760,9 +1787,9 @@ void Monitor::handle_probe_probe(MonOpRequestRef op) // did we discover a peer here? if (!monmap->contains(m->get_source_addr())) { - dout(1) << " adding peer " << m->get_source_addr() + dout(1) << " adding peer " << m->get_source_addrs() << " to list of hints" << dendl; - extra_probe_peers.insert(m->get_source_addr()); + extra_probe_peers.insert(m->get_source_addrs()); } out: @@ -1841,7 +1868,7 @@ void Monitor::handle_probe_reply(MonOpRequestRef op) return; } - entity_inst_t other = m->get_source_inst(); + entity_addrvec_t other = m->get_source_addrs(); if (m->paxos_last_version < sync_last_committed_floor) { dout(10) << " peer paxos versions [" << m->paxos_first_version @@ -2416,10 +2443,11 @@ void Monitor::get_mon_status(Formatter *f, ostream& ss) f->close_section(); // outside_quorum f->open_array_section("extra_probe_peers"); - for (set::iterator p = extra_probe_peers.begin(); + for (set::iterator p = extra_probe_peers.begin(); p != extra_probe_peers.end(); - ++p) - f->dump_stream("peer") << *p; + ++p) { + f->dump_object("peer", *p); + } f->close_section(); // extra_probe_peers f->open_array_section("sync_provider"); @@ -2427,7 +2455,7 @@ void Monitor::get_mon_status(Formatter *f, ostream& ss) p != sync_providers.end(); ++p) { f->dump_unsigned("cookie", p->second.cookie); - f->dump_stream("entity") << p->second.entity; + f->dump_object("addrs", p->second.addrs); f->dump_stream("timeout") << p->second.timeout; f->dump_unsigned("last_committed", p->second.last_committed); f->dump_stream("last_key") << p->second.last_key; diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index a2b17f77fe6c..f6ba72312583 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -149,7 +149,7 @@ public: MonMap *monmap; uuid_d fingerprint; - set extra_probe_peers; + set extra_probe_peers; LogClient log_client; LogChannelRef clog; @@ -302,7 +302,7 @@ private: * @} // provider state */ struct SyncProvider { - entity_inst_t entity; ///< who + entity_addrvec_t addrs; uint64_t cookie; ///< unique cookie for this sync attempt utime_t timeout; ///< when we give up and expire this attempt version_t last_committed; ///< last paxos version on peer @@ -324,7 +324,7 @@ private: /** * @} // requester state */ - entity_inst_t sync_provider; ///< who we are syncing from + entity_addrvec_t sync_provider; ///< who we are syncing from uint64_t sync_cookie; ///< 0 if we are starting, non-zero otherwise bool sync_full; ///< true if we are a full sync, false for recent catch-up version_t sync_start_version; ///< last_committed at sync start @@ -398,7 +398,7 @@ private: * @param entity where to pull committed state from * @param full whether to do a full sync or just catch up on recent paxos */ - void sync_start(entity_inst_t &entity, bool full); + void sync_start(entity_addrvec_t &addrs, bool full); public: /** diff --git a/src/tools/monmaptool.cc b/src/tools/monmaptool.cc index c551aa2d62d5..9ce4a3221a9b 100644 --- a/src/tools/monmaptool.cc +++ b/src/tools/monmaptool.cc @@ -355,7 +355,7 @@ int main(int argc, const char **argv) get_str_list(g_conf()->mon_initial_members, initial_members); if (!initial_members.empty()) { cout << "initial_members " << initial_members << ", filtering seed monmap" << std::endl; - set removed; + set removed; monmap.set_initial_members(g_ceph_context, initial_members, string(), entity_addrvec_t(), &removed); -- 2.47.3