From 18642ed351a303490c1b1b654ad3b60a8f7593dc Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@inktank.com>
Date: Wed, 9 Apr 2014 16:03:05 -0700
Subject: [PATCH] mon: tell peers missing features during probe

Use a new probe op to inform mons that they are missing features during
the earliest probe phase.  This prevents them from getting as far as
the sync entirely if they are too old.

We still need to refuse to speak to them if they try to call an election,
which they could do based on their replies from other peers.

Note that old clients will assert on getting a message type string they
don't understand, so we need to be careful not to send the probe reply
to older clients.  The feature bit we use is not precise in that it does
not cover recent dev releases, but it does work for dumpling and emperor.

Signed-off-by: Sage Weil <sage@inktank.com>
---
 src/messages/MMonProbe.h | 15 +++++++++++++--
 src/mon/Monitor.cc       | 31 ++++++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/src/messages/MMonProbe.h b/src/messages/MMonProbe.h
index 4d01d56e936d9..2f8a60b347c6b 100644
--- a/src/messages/MMonProbe.h
+++ b/src/messages/MMonProbe.h
@@ -22,7 +22,7 @@
 
 class MMonProbe : public Message {
 public:
-  static const int HEAD_VERSION = 5;
+  static const int HEAD_VERSION = 6;
   static const int COMPAT_VERSION = 5;
 
   enum {
@@ -31,6 +31,7 @@ public:
     OP_SLURP = 3,
     OP_SLURP_LATEST = 4,
     OP_DATA = 5,
+    OP_MISSING_FEATURES = 6,
   };
 
   static const char *get_opname(int o) {
@@ -40,6 +41,7 @@ public:
     case OP_SLURP: return "slurp";
     case OP_SLURP_LATEST: return "slurp_latest";
     case OP_DATA: return "data";
+    case OP_MISSING_FEATURES: return "missing_features";
     default: assert(0); return 0;
     }
   }
@@ -52,6 +54,7 @@ public:
   version_t paxos_first_version;
   version_t paxos_last_version;
   bool has_ever_joined;
+  uint64_t required_features;
 
   MMonProbe()
     : Message(MSG_MON_PROBE, HEAD_VERSION, COMPAT_VERSION) {}
@@ -62,7 +65,8 @@ public:
       name(n),
       paxos_first_version(0),
       paxos_last_version(0),
-      has_ever_joined(hej) {}
+      has_ever_joined(hej),
+      required_features(0) {}
 private:
   ~MMonProbe() {}
 
@@ -80,6 +84,8 @@ public:
     }
     if (!has_ever_joined)
       out << " new";
+    if (required_features)
+      out << " required_features " << required_features;
     out << ")";
   }
   
@@ -100,6 +106,7 @@ public:
     ::encode(has_ever_joined, payload);
     ::encode(paxos_first_version, payload);
     ::encode(paxos_last_version, payload);
+    ::encode(required_features, payload);
   }
   void decode_payload() {
     bufferlist::iterator p = payload.begin();
@@ -111,6 +118,10 @@ public:
     ::decode(has_ever_joined, p);
     ::decode(paxos_first_version, p);
     ::decode(paxos_last_version, p);
+    if (header.version >= 6)
+      ::decode(required_features, p);
+    else
+      required_features = 0;
   }
 };
 
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 890682829eed7..64be5ae64569d 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -1355,6 +1355,13 @@ void Monitor::handle_probe(MMonProbe *m)
     handle_probe_reply(m);
     break;
 
+  case MMonProbe::OP_MISSING_FEATURES:
+    derr << __func__ << " missing features, have " << CEPH_FEATURES_ALL
+	 << ", required " << required_features
+	 << ", missing " << (required_features & ~CEPH_FEATURES_ALL)
+	 << dendl;
+    break;
+
   default:
     m->put();
   }
@@ -1365,8 +1372,24 @@ void Monitor::handle_probe(MMonProbe *m)
  */
 void Monitor::handle_probe_probe(MMonProbe *m)
 {
-  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m << dendl;
-  MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY, name, has_ever_joined);
+  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m
+	   << " features " << m->get_connection()->get_features() << dendl;
+  uint64_t missing = required_features & ~m->get_connection()->get_features();
+  if (missing) {
+    dout(1) << " peer " << m->get_source_addr() << " missing features "
+	    << missing << dendl;
+    if (m->get_connection()->has_feature(CEPH_FEATURE_OSD_PRIMARY_AFFINITY)) {
+      MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_MISSING_FEATURES,
+				   name, has_ever_joined);
+      m->required_features = required_features;
+      messenger->send_message(r, m->get_connection());
+    }
+    m->put();
+    return;
+  }
+
+  MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY,
+			       name, has_ever_joined);
   r->name = name;
   r->quorum = quorum;
   monmap->encode(r->monmap_bl, m->get_connection()->get_features());
@@ -1376,9 +1399,11 @@ void Monitor::handle_probe_probe(MMonProbe *m)
 
   // did we discover a peer here?
   if (!monmap->contains(m->get_source_addr())) {
-    dout(1) << " adding peer " << m->get_source_addr() << " to list of hints" << dendl;
+    dout(1) << " adding peer " << m->get_source_addr()
+	    << " to list of hints" << dendl;
     extra_probe_peers.insert(m->get_source_addr());
   }
+
   m->put();
 }
 
-- 
2.39.5