mon: tell peers missing features during probe

Use a new probe op to inform mons that they are missing features during
the earliest probe phase.  This prevents them from getting as far as
the sync entirely if they are too old.

We still need to refuse to speak to them if they try to call an election,
which they could do based on their replies from other peers.

Note that old clients will assert on getting a message type string they
don't understand, so we need to be careful not to send the probe reply
to older clients.  The feature bit we use is not precise in that it does
not cover recent dev releases, but it does work for dumpling and emperor.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/messages/MMonProbe.h b/src/messages/MMonProbe.h
index 4d01d56e936d9c2559a96127aed5419192319ca0..2f8a60b347c6bdecd01c85de4532acb8ae4d5277 100644 (file)
--- a/src/messages/MMonProbe.h
+++ b/src/messages/MMonProbe.h
@@ -22,7 +22,7 @@
 
 class MMonProbe : public Message {
 public:
-  static const int HEAD_VERSION = 5;
+  static const int HEAD_VERSION = 6;
   static const int COMPAT_VERSION = 5;
 
   enum {
@@ -31,6 +31,7 @@ public:
     OP_SLURP = 3,
     OP_SLURP_LATEST = 4,
     OP_DATA = 5,
+    OP_MISSING_FEATURES = 6,
   };
 
   static const char *get_opname(int o) {
@@ -40,6 +41,7 @@ public:
     case OP_SLURP: return "slurp";
     case OP_SLURP_LATEST: return "slurp_latest";
     case OP_DATA: return "data";
+    case OP_MISSING_FEATURES: return "missing_features";
     default: assert(0); return 0;
     }
   }
@@ -52,6 +54,7 @@ public:
   version_t paxos_first_version;
   version_t paxos_last_version;
   bool has_ever_joined;
+  uint64_t required_features;
 
   MMonProbe()
     : Message(MSG_MON_PROBE, HEAD_VERSION, COMPAT_VERSION) {}
@@ -62,7 +65,8 @@ public:
       name(n),
       paxos_first_version(0),
       paxos_last_version(0),
-      has_ever_joined(hej) {}
+      has_ever_joined(hej),
+      required_features(0) {}
 private:
   ~MMonProbe() {}
 
@@ -80,6 +84,8 @@ public:
     }
     if (!has_ever_joined)
       out << " new";
+    if (required_features)
+      out << " required_features " << required_features;
     out << ")";
   }
   
@@ -100,6 +106,7 @@ public:
     ::encode(has_ever_joined, payload);
     ::encode(paxos_first_version, payload);
     ::encode(paxos_last_version, payload);
+    ::encode(required_features, payload);
   }
   void decode_payload() {
     bufferlist::iterator p = payload.begin();
@@ -111,6 +118,10 @@ public:
     ::decode(has_ever_joined, p);
     ::decode(paxos_first_version, p);
     ::decode(paxos_last_version, p);
+    if (header.version >= 6)
+      ::decode(required_features, p);
+    else
+      required_features = 0;
   }
 };
 

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 890682829eed755d48f6540e962284765b11eca1..64be5ae64569d22fd625ce7cb6d0bfbc18fb1990 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -1355,6 +1355,13 @@ void Monitor::handle_probe(MMonProbe *m)
     handle_probe_reply(m);
     break;
 
+  case MMonProbe::OP_MISSING_FEATURES:
+    derr << __func__ << " missing features, have " << CEPH_FEATURES_ALL
+        << ", required " << required_features
+        << ", missing " << (required_features & ~CEPH_FEATURES_ALL)
+        << dendl;
+    break;
+
   default:
     m->put();
   }
@@ -1365,8 +1372,24 @@ void Monitor::handle_probe(MMonProbe *m)
  */
 void Monitor::handle_probe_probe(MMonProbe *m)
 {
-  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m << dendl;
-  MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY, name, has_ever_joined);
+  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m
+          << " features " << m->get_connection()->get_features() << dendl;
+  uint64_t missing = required_features & ~m->get_connection()->get_features();
+  if (missing) {
+    dout(1) << " peer " << m->get_source_addr() << " missing features "
+           << missing << dendl;
+    if (m->get_connection()->has_feature(CEPH_FEATURE_OSD_PRIMARY_AFFINITY)) {
+      MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_MISSING_FEATURES,
+                                  name, has_ever_joined);
+      m->required_features = required_features;
+      messenger->send_message(r, m->get_connection());
+    }
+    m->put();
+    return;
+  }
+
+  MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY,
+                              name, has_ever_joined);
   r->name = name;
   r->quorum = quorum;
   monmap->encode(r->monmap_bl, m->get_connection()->get_features());
@@ -1376,9 +1399,11 @@ void Monitor::handle_probe_probe(MMonProbe *m)
 
   // did we discover a peer here?
   if (!monmap->contains(m->get_source_addr())) {
-    dout(1) << " adding peer " << m->get_source_addr() << " to list of hints" << dendl;
+    dout(1) << " adding peer " << m->get_source_addr()
+           << " to list of hints" << dendl;
     extra_probe_peers.insert(m->get_source_addr());
   }
+
   m->put();
 }