From a39813837ca84434e57afa47138973268ff366d3 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Sat, 21 Oct 2017 23:35:47 +0800 Subject: [PATCH] mgr: request daemon's metadata when receiving a report message from an unknown server Fixes: http://tracker.ceph.com/issues/21687 Signed-off-by: Chang Liu --- src/mgr/DaemonServer.cc | 54 +++++++++++++--- src/mgr/Mgr.cc | 134 ++++++++++++++++------------------------ src/mgr/Mgr.h | 29 +++++++++ 3 files changed, 130 insertions(+), 87 deletions(-) diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index bdb3cc96a7c1d..450c0059c0c99 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -12,7 +12,9 @@ */ #include "DaemonServer.h" +#include "mgr/Mgr.h" +#include "include/stringify.h" #include "include/str_list.h" #include "auth/RotatingKeyRing.h" #include "json_spirit/json_spirit_writer.h" @@ -418,15 +420,53 @@ bool DaemonServer::handle_report(MMgrReport *m) daemon = daemon_state.get(key); } else { // we don't know the hostname at this stage, reject MMgrReport here. - dout(1) << "rejecting report from " << key << ", since we do not have its metadata now." + dout(5) << "rejecting report from " << key << ", since we do not have its metadata now." << dendl; - // kill session - MgrSessionRef session(static_cast(m->get_connection()->get_priv())); - if (!session) { - return false; + + // issue metadata request in background + if (!daemon_state.is_updating(key) && + (key.first == "osd" || key.first == "mds")) { + + std::ostringstream oss; + auto c = new MetadataUpdate(daemon_state, key); + if (key.first == "osd") { + oss << "{\"prefix\": \"osd metadata\", \"id\": " + << key.second<< "}"; + + } else if (key.first == "mds") { + c->set_default("addr", stringify(m->get_source_addr())); + oss << "{\"prefix\": \"mds metadata\", \"who\": \"" + << key.second << "\"}"; + + } else { + ceph_abort(); + } + + monc->start_mon_command({oss.str()}, {}, &c->outbl, &c->outs, c); + } + + { + Mutex::Locker l(lock); + // kill session + MgrSessionRef session(static_cast(m->get_connection()->get_priv())); + if (!session) { + return false; + } + m->get_connection()->mark_down(); + session->put(); + + dout(10) << "unregistering osd." << session->osd_id + << " session " << session << " con " << m->get_connection() << dendl; + + if (osd_cons.find(session->osd_id) != osd_cons.end()) { + osd_cons[session->osd_id].erase(m->get_connection()); + } + + auto iter = daemon_connections.find(m->get_connection()); + if (iter != daemon_connections.end()) { + daemon_connections.erase(iter); + } } - m->get_connection()->mark_down(); - session->put(); return false; } diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index de60b4c771fa9..e359aee7f229d 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -68,98 +68,72 @@ Mgr::~Mgr() { } - -/** - * Context for completion of metadata mon commands: take - * the result and stash it in DaemonStateIndex - */ -class MetadataUpdate : public Context +void MetadataUpdate::finish(int r) { - DaemonStateIndex &daemon_state; - DaemonKey key; - - std::map defaults; - -public: - bufferlist outbl; - std::string outs; - - MetadataUpdate(DaemonStateIndex &daemon_state_, const DaemonKey &key_) - : daemon_state(daemon_state_), key(key_) {} - - void set_default(const std::string &k, const std::string &v) - { - defaults[k] = v; - } - - void finish(int r) override - { - daemon_state.clear_updating(key); - if (r == 0) { - if (key.first == "mds" || key.first == "osd") { - json_spirit::mValue json_result; - bool read_ok = json_spirit::read( - outbl.to_str(), json_result); - if (!read_ok) { - dout(1) << "mon returned invalid JSON for " - << key.first << "." << key.second << dendl; - return; - } - dout(4) << "mon returned valid metadata JSON for " + daemon_state.clear_updating(key); + if (r == 0) { + if (key.first == "mds" || key.first == "osd") { + json_spirit::mValue json_result; + bool read_ok = json_spirit::read( + outbl.to_str(), json_result); + if (!read_ok) { + dout(1) << "mon returned invalid JSON for " << key.first << "." << key.second << dendl; + return; + } + dout(4) << "mon returned valid metadata JSON for " + << key.first << "." << key.second << dendl; - json_spirit::mObject daemon_meta = json_result.get_obj(); + json_spirit::mObject daemon_meta = json_result.get_obj(); - // Apply any defaults - for (const auto &i : defaults) { - if (daemon_meta.find(i.first) == daemon_meta.end()) { - daemon_meta[i.first] = i.second; - } + // Apply any defaults + for (const auto &i : defaults) { + if (daemon_meta.find(i.first) == daemon_meta.end()) { + daemon_meta[i.first] = i.second; } + } - DaemonStatePtr state; - if (daemon_state.exists(key)) { - state = daemon_state.get(key); - Mutex::Locker l(state->lock); - if (key.first == "mds") { - daemon_meta.erase("name"); - } else if (key.first == "osd") { - daemon_meta.erase("id"); - } - daemon_meta.erase("hostname"); - state->metadata.clear(); - for (const auto &i : daemon_meta) { - state->metadata[i.first] = i.second.get_str(); - } - } else { - state = std::make_shared(daemon_state.types); - state->key = key; - state->hostname = daemon_meta.at("hostname").get_str(); - - if (key.first == "mds") { - daemon_meta.erase("name"); - } else if (key.first == "osd") { - daemon_meta.erase("id"); - } - daemon_meta.erase("hostname"); - - for (const auto &i : daemon_meta) { - state->metadata[i.first] = i.second.get_str(); - } - - daemon_state.insert(state); + DaemonStatePtr state; + if (daemon_state.exists(key)) { + state = daemon_state.get(key); + Mutex::Locker l(state->lock); + if (key.first == "mds") { + daemon_meta.erase("name"); + } else if (key.first == "osd") { + daemon_meta.erase("id"); + } + daemon_meta.erase("hostname"); + state->metadata.clear(); + for (const auto &i : daemon_meta) { + state->metadata[i.first] = i.second.get_str(); } } else { - ceph_abort(); + state = std::make_shared(daemon_state.types); + state->key = key; + state->hostname = daemon_meta.at("hostname").get_str(); + + if (key.first == "mds") { + daemon_meta.erase("name"); + } else if (key.first == "osd") { + daemon_meta.erase("id"); + } + daemon_meta.erase("hostname"); + + for (const auto &i : daemon_meta) { + state->metadata[i.first] = i.second.get_str(); + } + + daemon_state.insert(state); } } else { - dout(1) << "mon failed to return metadata for " - << key.first << "." << key.second << ": " - << cpp_strerror(r) << dendl; + ceph_abort(); } + } else { + dout(1) << "mon failed to return metadata for " + << key.first << "." << key.second << ": " + << cpp_strerror(r) << dendl; } -}; - +} void Mgr::background_init(Context *completion) { diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index 68f2b40b4616d..b1e6d064946e2 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -102,4 +102,33 @@ public: std::vector get_command_set() const; }; +/** + * Context for completion of metadata mon commands: take + * the result and stash it in DaemonStateIndex + */ +class MetadataUpdate : public Context +{ + +private: + DaemonStateIndex &daemon_state; + DaemonKey key; + + std::map defaults; + +public: + bufferlist outbl; + std::string outs; + + MetadataUpdate(DaemonStateIndex &daemon_state_, const DaemonKey &key_) + : daemon_state(daemon_state_), key(key_) {} + + void set_default(const std::string &k, const std::string &v) + { + defaults[k] = v; + } + + void finish(int r) override; +}; + + #endif -- 2.39.5