From 264807c157301c5d0e8172b7b22b5ea9806c8f08 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 24 May 2018 09:26:18 -0500 Subject: [PATCH] mon: change timecheck_* key to int We don't need to use entity_inst_t here since we're only paying attention to mons in the quorum. This involves changing the message format. To save some time/effort and complexity, we only support time checks on all-nautilus. If you are mid-upgrade, the time sync checks are skipped. Signed-off-by: Sage Weil --- src/messages/MTimeCheck2.h | 86 ++++++++++++++++++++++++++++++++++++++ src/mon/HealthMonitor.cc | 12 +++--- src/mon/Monitor.cc | 75 +++++++++++++++++---------------- src/mon/Monitor.h | 8 ++-- src/msg/Message.cc | 4 ++ src/msg/Message.h | 1 + 6 files changed, 138 insertions(+), 48 deletions(-) create mode 100644 src/messages/MTimeCheck2.h diff --git a/src/messages/MTimeCheck2.h b/src/messages/MTimeCheck2.h new file mode 100644 index 0000000000000..ea4f66ec8878a --- /dev/null +++ b/src/messages/MTimeCheck2.h @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2012 Inktank, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +struct MTimeCheck2 : public Message +{ + static const int HEAD_VERSION = 1; + static const int COMPAT_VERSION = 1; + + enum { + OP_PING = 1, + OP_PONG = 2, + OP_REPORT = 3, + }; + + int op = 0; + version_t epoch = 0; + version_t round = 0; + + utime_t timestamp; + map skews; + map latencies; + + MTimeCheck2() : Message(MSG_TIMECHECK2, HEAD_VERSION, COMPAT_VERSION) { } + MTimeCheck2(int op) : + Message(MSG_TIMECHECK2, HEAD_VERSION, COMPAT_VERSION), + op(op) + { } + +private: + ~MTimeCheck2() override { } + +public: + const char *get_type_name() const override { return "time_check2"; } + const char *get_op_name() const { + switch (op) { + case OP_PING: return "ping"; + case OP_PONG: return "pong"; + case OP_REPORT: return "report"; + } + return "???"; + } + void print(ostream &o) const override { + o << "time_check( " << get_op_name() + << " e " << epoch << " r " << round; + if (op == OP_PONG) { + o << " ts " << timestamp; + } else if (op == OP_REPORT) { + o << " #skews " << skews.size() + << " #latencies " << latencies.size(); + } + o << " )"; + } + + void decode_payload() override { + auto p = payload.cbegin(); + decode(op, p); + decode(epoch, p); + decode(round, p); + decode(timestamp, p); + decode(skews, p); + decode(latencies, p); + } + + void encode_payload(uint64_t features) override { + using ceph::encode; + encode(op, payload); + encode(epoch, payload); + encode(round, payload); + encode(timestamp, payload); + encode(skews, payload, features); + encode(latencies, payload, features); + } +}; diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc index e4fcdb8327db7..8682983ab7865 100644 --- a/src/mon/HealthMonitor.cc +++ b/src/mon/HealthMonitor.cc @@ -339,19 +339,17 @@ bool HealthMonitor::check_leader_health() if (!mon->timecheck_skews.empty()) { list warns; list details; - for (map::iterator i = mon->timecheck_skews.begin(); - i != mon->timecheck_skews.end(); ++i) { - entity_inst_t inst = i->first; - double skew = i->second; - double latency = mon->timecheck_latencies[inst]; - string name = mon->monmap->get_name(inst.addr); + for (auto& i : mon->timecheck_skews) { + double skew = i.second; + double latency = mon->timecheck_latencies[i.first]; + string name = mon->monmap->get_name(i.first); ostringstream tcss; health_status_t tcstatus = mon->timecheck_status(tcss, skew, latency); if (tcstatus != HEALTH_OK) { warns.push_back(name); ostringstream tmp_ss; tmp_ss << "mon." << name - << " addr " << inst.addr << " " << tcss.str() + << " " << tcss.str() << " (latency " << latency << "s)"; details.push_back(tmp_ss.str()); } diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 2a8df86004e1f..5e8604c71a99e 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -51,7 +51,7 @@ #include "messages/MAuthReply.h" -#include "messages/MTimeCheck.h" +#include "messages/MTimeCheck2.h" #include "messages/MPing.h" #include "common/strtol.h" @@ -3233,10 +3233,9 @@ void Monitor::handle_command(MonOpRequestRef op) if (!timecheck_skews.empty()) { f->open_object_section("time_skew_status"); for (auto& i : timecheck_skews) { - entity_inst_t inst = i.first; double skew = i.second; - double latency = timecheck_latencies[inst]; - string name = monmap->get_name(inst.addr); + double latency = timecheck_latencies[i.first]; + string name = monmap->get_name(i.first); ostringstream tcss; health_status_t tcstatus = timecheck_status(tcss, skew, latency); f->open_object_section(name.c_str()); @@ -4322,6 +4321,9 @@ void Monitor::dispatch_op(MonOpRequestRef op) return; case MSG_TIMECHECK: + dout(5) << __func__ << " ignoring " << op << dendl; + return; + case MSG_TIMECHECK2: handle_timecheck(op); return; @@ -4367,7 +4369,10 @@ void Monitor::timecheck_start() { dout(10) << __func__ << dendl; timecheck_cleanup(); - timecheck_start_round(); + if (get_quorum_mon_features().contains_all( + ceph::features::mon::FEATURE_NAUTILUS)) { + timecheck_start_round(); + } } void Monitor::timecheck_finish() @@ -4429,9 +4434,8 @@ void Monitor::timecheck_finish_round(bool success) dout(10) << __func__ << " " << timecheck_waiting.size() << " peers still waiting:"; - for (map::iterator p = timecheck_waiting.begin(); - p != timecheck_waiting.end(); ++p) { - *_dout << " " << p->first.name; + for (auto& p : timecheck_waiting) { + *_dout << " mon." << p.first; } *_dout << dendl; timecheck_waiting.clear(); @@ -4498,13 +4502,11 @@ void Monitor::timecheck_check_skews() assert(timecheck_latencies.size() == timecheck_skews.size()); bool found_skew = false; - for (map::iterator p = timecheck_skews.begin(); - p != timecheck_skews.end(); ++p) { - + for (auto& p : timecheck_skews) { double abs_skew; - if (timecheck_has_skew(p->second, &abs_skew)) { + if (timecheck_has_skew(p.second, &abs_skew)) { dout(10) << __func__ - << " " << p->first << " skew " << abs_skew << dendl; + << " " << p.first << " skew " << abs_skew << dendl; found_skew = true; } } @@ -4541,27 +4543,26 @@ void Monitor::timecheck_report() if (monmap->get_name(*q) == name) continue; - MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_REPORT); + MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_REPORT); m->epoch = get_epoch(); m->round = timecheck_round; - for (map::iterator it = timecheck_skews.begin(); - it != timecheck_skews.end(); ++it) { - double skew = it->second; - double latency = timecheck_latencies[it->first]; + for (auto& it : timecheck_skews) { + double skew = it.second; + double latency = timecheck_latencies[it.first]; - m->skews[it->first] = skew; - m->latencies[it->first] = latency; + m->skews[it.first] = skew; + m->latencies[it.first] = latency; if (do_output) { - dout(25) << __func__ << " " << it->first + dout(25) << __func__ << " mon." << it.first << " latency " << latency << " skew " << skew << dendl; } } do_output = false; + dout(10) << __func__ << " send report to mon." << *q << dendl; entity_inst_t inst = monmap->get_inst(*q); - dout(10) << __func__ << " send report to " << inst << dendl; messenger->send_message(m, inst); } } @@ -4582,8 +4583,8 @@ void Monitor::timecheck() << " round " << timecheck_round << dendl; // we are at the eye of the storm; the point of reference - timecheck_skews[messenger->get_myinst()] = 0.0; - timecheck_latencies[messenger->get_myinst()] = 0.0; + timecheck_skews[rank] = 0.0; + timecheck_latencies[rank] = 0.0; for (set::iterator it = quorum.begin(); it != quorum.end(); ++it) { if (monmap->get_name(*it) == name) @@ -4591,11 +4592,11 @@ void Monitor::timecheck() entity_inst_t inst = monmap->get_inst(*it); utime_t curr_time = ceph_clock_now(); - timecheck_waiting[inst] = curr_time; - MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_PING); + timecheck_waiting[*it] = curr_time; + MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_PING); m->epoch = get_epoch(); m->round = timecheck_round; - dout(10) << __func__ << " send " << *m << " to " << inst << dendl; + dout(10) << __func__ << " send " << *m << " to mon." << *it << dendl; messenger->send_message(m, inst); } } @@ -4619,12 +4620,12 @@ health_status_t Monitor::timecheck_status(ostringstream &ss, void Monitor::handle_timecheck_leader(MonOpRequestRef op) { - MTimeCheck *m = static_cast(op->get_req()); + MTimeCheck2 *m = static_cast(op->get_req()); dout(10) << __func__ << " " << *m << dendl; /* handles PONG's */ - assert(m->op == MTimeCheck::OP_PONG); + assert(m->op == MTimeCheck2::OP_PONG); - entity_inst_t other = m->get_source_inst(); + int other = m->get_source().num(); if (m->epoch < get_epoch()) { dout(1) << __func__ << " got old timecheck epoch " << m->epoch << " from " << other @@ -4737,11 +4738,11 @@ void Monitor::handle_timecheck_leader(MonOpRequestRef op) void Monitor::handle_timecheck_peon(MonOpRequestRef op) { - MTimeCheck *m = static_cast(op->get_req()); + MTimeCheck2 *m = static_cast(op->get_req()); dout(10) << __func__ << " " << *m << dendl; assert(is_peon()); - assert(m->op == MTimeCheck::OP_PING || m->op == MTimeCheck::OP_REPORT); + assert(m->op == MTimeCheck2::OP_PING || m->op == MTimeCheck2::OP_REPORT); if (m->epoch != get_epoch()) { dout(1) << __func__ << " got wrong epoch " @@ -4759,7 +4760,7 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op) timecheck_round = m->round; - if (m->op == MTimeCheck::OP_REPORT) { + if (m->op == MTimeCheck2::OP_REPORT) { assert((timecheck_round % 2) == 0); timecheck_latencies.swap(m->latencies); timecheck_skews.swap(m->skews); @@ -4767,7 +4768,7 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op) } assert((timecheck_round % 2) != 0); - MTimeCheck *reply = new MTimeCheck(MTimeCheck::OP_PONG); + MTimeCheck2 *reply = new MTimeCheck2(MTimeCheck2::OP_PONG); utime_t curr_time = ceph_clock_now(); reply->timestamp = curr_time; reply->epoch = m->epoch; @@ -4779,17 +4780,17 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op) void Monitor::handle_timecheck(MonOpRequestRef op) { - MTimeCheck *m = static_cast(op->get_req()); + MTimeCheck2 *m = static_cast(op->get_req()); dout(10) << __func__ << " " << *m << dendl; if (is_leader()) { - if (m->op != MTimeCheck::OP_PONG) { + if (m->op != MTimeCheck2::OP_PONG) { dout(1) << __func__ << " drop unexpected msg (not pong)" << dendl; } else { handle_timecheck_leader(op); } } else if (is_peon()) { - if (m->op != MTimeCheck::OP_PING && m->op != MTimeCheck::OP_REPORT) { + if (m->op != MTimeCheck2::OP_PING && m->op != MTimeCheck2::OP_REPORT) { dout(1) << __func__ << " drop unexpected msg (not ping or report)" << dendl; } else { handle_timecheck_peon(op); diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 864a8f8289b8e..7d810926f9cb2 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -107,7 +107,7 @@ class MMonProbe; struct MMonSubscribe; struct MRoute; struct MForward; -struct MTimeCheck; +struct MTimeCheck2; struct MMonHealth; #define COMPAT_SET_LOC "feature_set" @@ -485,9 +485,9 @@ private: * - Once all the quorum members have pong'ed, the leader will share the * clock skew and latency maps with all the monitors in the quorum. */ - map timecheck_waiting; - map timecheck_skews; - map timecheck_latencies; + map timecheck_waiting; + map timecheck_skews; + map timecheck_latencies; // odd value means we are mid-round; even value means the round has // finished. version_t timecheck_round; diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 16201febb05e2..a360089d20998 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -182,6 +182,7 @@ #include "messages/MWatchNotify.h" #include "messages/MTimeCheck.h" +#include "messages/MTimeCheck2.h" #include "common/config.h" @@ -809,6 +810,9 @@ Message *decode_message(CephContext *cct, int crcflags, case MSG_TIMECHECK: m = new MTimeCheck(); break; + case MSG_TIMECHECK2: + m = new MTimeCheck2(); + break; case MSG_MON_HEALTH: m = new MMonHealth(); diff --git a/src/msg/Message.h b/src/msg/Message.h index d74590efaae78..2aa5f1be36519 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -194,6 +194,7 @@ #define MSG_NOP 0x607 #define MSG_MON_HEALTH_CHECKS 0x608 +#define MSG_TIMECHECK2 0x609 // *** ceph-mgr <-> OSD/MDS daemons *** #define MSG_MGR_OPEN 0x700 -- 2.39.5