mon: change timecheck_* key to int

We don't need to use entity_inst_t here since we're only paying attention
to mons in the quorum.

This involves changing the message format.  To save some time/effort and
complexity, we only support time checks on all-nautilus.  If you are
mid-upgrade, the time sync checks are skipped.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/messages/MTimeCheck2.h b/src/messages/MTimeCheck2.h
new file mode 100644 (file)
index 0000000..ea4f66e
--- /dev/null
+++ b/src/messages/MTimeCheck2.h
@@ -0,0 +1,86 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+struct MTimeCheck2 : public Message
+{
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+  enum {
+    OP_PING = 1,
+    OP_PONG = 2,
+    OP_REPORT = 3,
+  };
+
+  int op = 0;
+  version_t epoch = 0;
+  version_t round = 0;
+
+  utime_t timestamp;
+  map<int, double> skews;
+  map<int, double> latencies;
+
+  MTimeCheck2() : Message(MSG_TIMECHECK2, HEAD_VERSION, COMPAT_VERSION) { }
+  MTimeCheck2(int op) :
+    Message(MSG_TIMECHECK2, HEAD_VERSION, COMPAT_VERSION),
+    op(op)
+  { }
+
+private:
+  ~MTimeCheck2() override { }
+
+public:
+  const char *get_type_name() const override { return "time_check2"; }
+  const char *get_op_name() const {
+    switch (op) {
+    case OP_PING: return "ping";
+    case OP_PONG: return "pong";
+    case OP_REPORT: return "report";
+    }
+    return "???";
+  }
+  void print(ostream &o) const override {
+    o << "time_check( " << get_op_name()
+      << " e " << epoch << " r " << round;
+    if (op == OP_PONG) {
+      o << " ts " << timestamp;
+    } else if (op == OP_REPORT) {
+      o << " #skews " << skews.size()
+        << " #latencies " << latencies.size();
+    }
+    o << " )";
+  }
+
+  void decode_payload() override {
+    auto p = payload.cbegin();
+    decode(op, p);
+    decode(epoch, p);
+    decode(round, p);
+    decode(timestamp, p);
+    decode(skews, p);
+    decode(latencies, p);
+  }
+
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    encode(op, payload);
+    encode(epoch, payload);
+    encode(round, payload);
+    encode(timestamp, payload);
+    encode(skews, payload, features);
+    encode(latencies, payload, features);
+  }
+};

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index e4fcdb8327db78c58196bf8a8717f6fbd284275b..8682983ab7865a2aeff3a7ffaac76da00d075b08 100644 (file)
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -339,19 +339,17 @@ bool HealthMonitor::check_leader_health()
   if (!mon->timecheck_skews.empty()) {
     list<string> warns;
     list<string> details;
-    for (map<entity_inst_t,double>::iterator i = mon->timecheck_skews.begin();
-        i != mon->timecheck_skews.end(); ++i) {
-      entity_inst_t inst = i->first;
-      double skew = i->second;
-      double latency = mon->timecheck_latencies[inst];
-      string name = mon->monmap->get_name(inst.addr);
+    for (auto& i : mon->timecheck_skews) {
+      double skew = i.second;
+      double latency = mon->timecheck_latencies[i.first];
+      string name = mon->monmap->get_name(i.first);
       ostringstream tcss;
       health_status_t tcstatus = mon->timecheck_status(tcss, skew, latency);
       if (tcstatus != HEALTH_OK) {
        warns.push_back(name);
        ostringstream tmp_ss;
        tmp_ss << "mon." << name
-              << " addr " << inst.addr << " " << tcss.str()
+              << " " << tcss.str()
               << " (latency " << latency << "s)";
        details.push_back(tmp_ss.str());
       }

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 2a8df86004e1f8861b4cfccd07c83d523786c141..5e8604c71a99e090935a2e0abccd79296c271bef 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -51,7 +51,7 @@
 
 #include "messages/MAuthReply.h"
 
-#include "messages/MTimeCheck.h"
+#include "messages/MTimeCheck2.h"
 #include "messages/MPing.h"
 
 #include "common/strtol.h"
@@ -3233,10 +3233,9 @@ void Monitor::handle_command(MonOpRequestRef op)
     if (!timecheck_skews.empty()) {
       f->open_object_section("time_skew_status");
       for (auto& i : timecheck_skews) {
-       entity_inst_t inst = i.first;
        double skew = i.second;
-       double latency = timecheck_latencies[inst];
-       string name = monmap->get_name(inst.addr);
+       double latency = timecheck_latencies[i.first];
+       string name = monmap->get_name(i.first);
        ostringstream tcss;
        health_status_t tcstatus = timecheck_status(tcss, skew, latency);
        f->open_object_section(name.c_str());
@@ -4322,6 +4321,9 @@ void Monitor::dispatch_op(MonOpRequestRef op)
       return;
 
     case MSG_TIMECHECK:
+      dout(5) << __func__ << " ignoring " << op << dendl;
+      return;
+    case MSG_TIMECHECK2:
       handle_timecheck(op);
       return;
 
@@ -4367,7 +4369,10 @@ void Monitor::timecheck_start()
 {
   dout(10) << __func__ << dendl;
   timecheck_cleanup();
-  timecheck_start_round();
+  if (get_quorum_mon_features().contains_all(
+       ceph::features::mon::FEATURE_NAUTILUS)) {
+    timecheck_start_round();
+  }
 }
 
 void Monitor::timecheck_finish()
@@ -4429,9 +4434,8 @@ void Monitor::timecheck_finish_round(bool success)
 
   dout(10) << __func__ << " " << timecheck_waiting.size()
            << " peers still waiting:";
-  for (map<entity_inst_t,utime_t>::iterator p = timecheck_waiting.begin();
-      p != timecheck_waiting.end(); ++p) {
-    *_dout << " " << p->first.name;
+  for (auto& p : timecheck_waiting) {
+    *_dout << " mon." << p.first;
   }
   *_dout << dendl;
   timecheck_waiting.clear();
@@ -4498,13 +4502,11 @@ void Monitor::timecheck_check_skews()
   assert(timecheck_latencies.size() == timecheck_skews.size());
 
   bool found_skew = false;
-  for (map<entity_inst_t, double>::iterator p = timecheck_skews.begin();
-       p != timecheck_skews.end(); ++p) {
-
+  for (auto& p : timecheck_skews) {
     double abs_skew;
-    if (timecheck_has_skew(p->second, &abs_skew)) {
+    if (timecheck_has_skew(p.second, &abs_skew)) {
       dout(10) << __func__
-               << " " << p->first << " skew " << abs_skew << dendl;
+               << " " << p.first << " skew " << abs_skew << dendl;
       found_skew = true;
     }
   }
@@ -4541,27 +4543,26 @@ void Monitor::timecheck_report()
     if (monmap->get_name(*q) == name)
       continue;
 
-    MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_REPORT);
+    MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_REPORT);
     m->epoch = get_epoch();
     m->round = timecheck_round;
 
-    for (map<entity_inst_t, double>::iterator it = timecheck_skews.begin();
-         it != timecheck_skews.end(); ++it) {
-      double skew = it->second;
-      double latency = timecheck_latencies[it->first];
+    for (auto& it : timecheck_skews) {
+      double skew = it.second;
+      double latency = timecheck_latencies[it.first];
 
-      m->skews[it->first] = skew;
-      m->latencies[it->first] = latency;
+      m->skews[it.first] = skew;
+      m->latencies[it.first] = latency;
 
       if (do_output) {
-        dout(25) << __func__ << " " << it->first
+        dout(25) << __func__ << " mon." << it.first
                  << " latency " << latency
                  << " skew " << skew << dendl;
       }
     }
     do_output = false;
+    dout(10) << __func__ << " send report to mon." << *q << dendl;
     entity_inst_t inst = monmap->get_inst(*q);
-    dout(10) << __func__ << " send report to " << inst << dendl;
     messenger->send_message(m, inst);
   }
 }
@@ -4582,8 +4583,8 @@ void Monitor::timecheck()
            << " round " << timecheck_round << dendl;
 
   // we are at the eye of the storm; the point of reference
-  timecheck_skews[messenger->get_myinst()] = 0.0;
-  timecheck_latencies[messenger->get_myinst()] = 0.0;
+  timecheck_skews[rank] = 0.0;
+  timecheck_latencies[rank] = 0.0;
 
   for (set<int>::iterator it = quorum.begin(); it != quorum.end(); ++it) {
     if (monmap->get_name(*it) == name)
@@ -4591,11 +4592,11 @@ void Monitor::timecheck()
 
     entity_inst_t inst = monmap->get_inst(*it);
     utime_t curr_time = ceph_clock_now();
-    timecheck_waiting[inst] = curr_time;
-    MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_PING);
+    timecheck_waiting[*it] = curr_time;
+    MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_PING);
     m->epoch = get_epoch();
     m->round = timecheck_round;
-    dout(10) << __func__ << " send " << *m << " to " << inst << dendl;
+    dout(10) << __func__ << " send " << *m << " to mon." << *it << dendl;
     messenger->send_message(m, inst);
   }
 }
@@ -4619,12 +4620,12 @@ health_status_t Monitor::timecheck_status(ostringstream &ss,
 
 void Monitor::handle_timecheck_leader(MonOpRequestRef op)
 {
-  MTimeCheck *m = static_cast<MTimeCheck*>(op->get_req());
+  MTimeCheck2 *m = static_cast<MTimeCheck2*>(op->get_req());
   dout(10) << __func__ << " " << *m << dendl;
   /* handles PONG's */
-  assert(m->op == MTimeCheck::OP_PONG);
+  assert(m->op == MTimeCheck2::OP_PONG);
 
-  entity_inst_t other = m->get_source_inst();
+  int other = m->get_source().num();
   if (m->epoch < get_epoch()) {
     dout(1) << __func__ << " got old timecheck epoch " << m->epoch
             << " from " << other
@@ -4737,11 +4738,11 @@ void Monitor::handle_timecheck_leader(MonOpRequestRef op)
 
 void Monitor::handle_timecheck_peon(MonOpRequestRef op)
 {
-  MTimeCheck *m = static_cast<MTimeCheck*>(op->get_req());
+  MTimeCheck2 *m = static_cast<MTimeCheck2*>(op->get_req());
   dout(10) << __func__ << " " << *m << dendl;
 
   assert(is_peon());
-  assert(m->op == MTimeCheck::OP_PING || m->op == MTimeCheck::OP_REPORT);
+  assert(m->op == MTimeCheck2::OP_PING || m->op == MTimeCheck2::OP_REPORT);
 
   if (m->epoch != get_epoch()) {
     dout(1) << __func__ << " got wrong epoch "
@@ -4759,7 +4760,7 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op)
 
   timecheck_round = m->round;
 
-  if (m->op == MTimeCheck::OP_REPORT) {
+  if (m->op == MTimeCheck2::OP_REPORT) {
     assert((timecheck_round % 2) == 0);
     timecheck_latencies.swap(m->latencies);
     timecheck_skews.swap(m->skews);
@@ -4767,7 +4768,7 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op)
   }
 
   assert((timecheck_round % 2) != 0);
-  MTimeCheck *reply = new MTimeCheck(MTimeCheck::OP_PONG);
+  MTimeCheck2 *reply = new MTimeCheck2(MTimeCheck2::OP_PONG);
   utime_t curr_time = ceph_clock_now();
   reply->timestamp = curr_time;
   reply->epoch = m->epoch;
@@ -4779,17 +4780,17 @@ void Monitor::handle_timecheck_peon(MonOpRequestRef op)
 
 void Monitor::handle_timecheck(MonOpRequestRef op)
 {
-  MTimeCheck *m = static_cast<MTimeCheck*>(op->get_req());
+  MTimeCheck2 *m = static_cast<MTimeCheck2*>(op->get_req());
   dout(10) << __func__ << " " << *m << dendl;
 
   if (is_leader()) {
-    if (m->op != MTimeCheck::OP_PONG) {
+    if (m->op != MTimeCheck2::OP_PONG) {
       dout(1) << __func__ << " drop unexpected msg (not pong)" << dendl;
     } else {
       handle_timecheck_leader(op);
     }
   } else if (is_peon()) {
-    if (m->op != MTimeCheck::OP_PING && m->op != MTimeCheck::OP_REPORT) {
+    if (m->op != MTimeCheck2::OP_PING && m->op != MTimeCheck2::OP_REPORT) {
       dout(1) << __func__ << " drop unexpected msg (not ping or report)" << dendl;
     } else {
       handle_timecheck_peon(op);

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 864a8f8289b8eb37a25c345d1ebb18fb0a71e8fa..7d810926f9cb27f1f148a06d3e5a3e668952224c 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -107,7 +107,7 @@ class MMonProbe;
 struct MMonSubscribe;
 struct MRoute;
 struct MForward;
-struct MTimeCheck;
+struct MTimeCheck2;
 struct MMonHealth;
 
 #define COMPAT_SET_LOC "feature_set"
@@ -485,9 +485,9 @@ private:
    *  - Once all the quorum members have pong'ed, the leader will share the
    *    clock skew and latency maps with all the monitors in the quorum.
    */
-  map<entity_inst_t, utime_t> timecheck_waiting;
-  map<entity_inst_t, double> timecheck_skews;
-  map<entity_inst_t, double> timecheck_latencies;
+  map<int, utime_t> timecheck_waiting;
+  map<int, double> timecheck_skews;
+  map<int, double> timecheck_latencies;
   // odd value means we are mid-round; even value means the round has
   // finished.
   version_t timecheck_round;

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 16201febb05e2e445964ad5542ed047364fe239f..a360089d209985f56fec7fa1e9b665184b3448c8 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -182,6 +182,7 @@
 
 #include "messages/MWatchNotify.h"
 #include "messages/MTimeCheck.h"
+#include "messages/MTimeCheck2.h"
 
 #include "common/config.h"
 
@@ -809,6 +810,9 @@ Message *decode_message(CephContext *cct, int crcflags,
   case MSG_TIMECHECK:
     m = new MTimeCheck();
     break;
+  case MSG_TIMECHECK2:
+    m = new MTimeCheck2();
+    break;
 
   case MSG_MON_HEALTH:
     m = new MMonHealth();

diff --git a/src/msg/Message.h b/src/msg/Message.h
index d74590efaae78a53db5821c02f59cf1f74412a02..2aa5f1be365193ec930830135d4471c3b1930935 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -194,6 +194,7 @@
 #define MSG_NOP                   0x607
 
 #define MSG_MON_HEALTH_CHECKS     0x608
+#define MSG_TIMECHECK2            0x609
 
 // *** ceph-mgr <-> OSD/MDS daemons ***
 #define MSG_MGR_OPEN              0x700