mon: Elector: support mon-specific features

Signed-off-by: Joao Eduardo Luis <joao@suse.de>

diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc
index b880929777b6ab4f14dd9a311c60215588e8e290..07e0533a057ebfb2198e253979f9e7d6d07c2331 100644 (file)
--- a/src/mon/Elector.cc
+++ b/src/mon/Elector.cc
@@ -88,13 +88,16 @@ void Elector::start()
   }
   start_stamp = ceph_clock_now(g_ceph_context);
   electing_me = true;
-  acked_me[mon->rank] = CEPH_FEATURES_ALL;
+  acked_me[mon->rank].cluster_features = CEPH_FEATURES_ALL;
+  acked_me[mon->rank].mon_features = ceph::features::mon::get_supported();
   leader_acked = -1;
 
   // bcast to everyone else
   for (unsigned i=0; i<mon->monmap->size(); ++i) {
     if ((int)i == mon->rank) continue;
-    Message *m = new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);
+    MMonElection *m =
+      new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);
+    m->mon_features = ceph::features::mon::get_supported();
     mon->messenger->send_message(m, mon->monmap->get_inst(i));
   }
   
@@ -116,6 +119,7 @@ void Elector::defer(int who)
   leader_acked = who;
   ack_stamp = ceph_clock_now(g_ceph_context);
   MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);
+  m->mon_features = ceph::features::mon::get_supported();
   m->sharing_bl = mon->get_supported_commands_bl();
   mon->messenger->send_message(m, mon->monmap->get_inst(who));
   
@@ -187,12 +191,15 @@ void Elector::victory()
   leader_acked = -1;
   electing_me = false;
 
-  uint64_t features = CEPH_FEATURES_ALL;
+  uint64_t cluster_features = CEPH_FEATURES_ALL;
+  mon_feature_t mon_features = ceph::features::mon::get_supported();
   set<int> quorum;
-  for (map<int, uint64_t>::iterator p = acked_me.begin(); p != acked_me.end();
+  for (map<int, elector_features_t>::iterator p = acked_me.begin();
+       p != acked_me.end();
        ++p) {
     quorum.insert(p->first);
-    features &= p->second;
+    cluster_features &= p->second.cluster_features;
+    mon_features &= p->second.mon_features;
   }
 
   // decide what command set we're supporting
@@ -224,13 +231,16 @@ void Elector::victory()
     if (*p == mon->rank) continue;
     MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch, mon->monmap);
     m->quorum = quorum;
-    m->quorum_features = features;
+    m->quorum_features = cluster_features;
+    m->mon_features = mon_features;
     m->sharing_bl = *cmds_bl;
     mon->messenger->send_message(m, mon->monmap->get_inst(*p));
   }
     
   // tell monitor
-  mon->win_election(epoch, quorum, features, cmds, cmdsize, &copy_classic_mons);
+  mon->win_election(epoch, quorum,
+                    cluster_features, mon_features,
+                    cmds, cmdsize, &copy_classic_mons);
 }
 
 
@@ -315,13 +325,24 @@ void Elector::handle_ack(MonOpRequestRef op)
            << " without required features" << dendl;
     return;
   }
-  
+
   if (electing_me) {
     // thanks
-    acked_me[from] = m->get_connection()->get_features();
+    acked_me[from].cluster_features = m->get_connection()->get_features();
+    acked_me[from].mon_features = m->mon_features;
     if (!m->sharing_bl.length())
       classic_mons.insert(from);
-    dout(5) << " so far i have " << acked_me << dendl;
+    dout(5) << " so far i have {";
+    for (map<int, elector_features_t>::const_iterator p = acked_me.begin();
+         p != acked_me.end();
+         ++p) {
+      if (p != acked_me.begin())
+        *_dout << ",";
+      *_dout << " mon." << p->first << ":"
+             << " features " << p->second.cluster_features
+             << " " << p->second.mon_features;
+    }
+    *_dout << " }" << dendl;
     
     // is that _everyone_?
     if (acked_me.size() == mon->monmap->size()) {
@@ -339,7 +360,10 @@ void Elector::handle_victory(MonOpRequestRef op)
 {
   op->mark_event("elector:handle_victory");
   MMonElection *m = static_cast<MMonElection*>(op->get_req());
-  dout(5) << "handle_victory from " << m->get_source() << " quorum_features " << m->quorum_features << dendl;
+  dout(5) << "handle_victory from " << m->get_source()
+          << " quorum_features " << m->quorum_features
+          << " " << m->mon_features
+          << dendl;
   int from = m->get_source().num();
 
   assert(from < mon->rank);
@@ -356,10 +380,11 @@ void Elector::handle_victory(MonOpRequestRef op)
   }
 
   bump_epoch(m->epoch);
-  
+
   // they win
-  mon->lose_election(epoch, m->quorum, from, m->quorum_features);
-  
+  mon->lose_election(epoch, m->quorum, from,
+                     m->quorum_features, m->mon_features);
+
   // cancel my timer
   cancel_timer();
 

diff --git a/src/mon/Elector.h b/src/mon/Elector.h
index a15445b81a2d2632d7a284f1b69105a3d89b3ecf..3ab347422ea39e5fceb2d9cbf5a793bac8d3ea45 100644 (file)
--- a/src/mon/Elector.h
+++ b/src/mon/Elector.h
@@ -22,6 +22,7 @@ using namespace std;
 #include "include/types.h"
 #include "include/Context.h"
 #include "mon/MonOpRequest.h"
+#include "mon/mon_types.h"
 
 class Monitor;
 
@@ -36,6 +37,25 @@ class Elector {
    * @{
    */
  private:
+   /**
+   * @defgroup Elector_h_internal_types Internal Types
+   * @{
+   */
+  /**
+   * This struct will hold the features from a given peer.
+   * Features may both be the cluster's (in the form of a uint64_t), or
+   * mon-specific features. Instead of keeping maps to hold them both, or
+   * a pair, which would be weird, a struct to keep them seems appropriate.
+   */
+  struct elector_features_t {
+    uint64_t cluster_features;
+    mon_feature_t mon_features;
+  };
+
+  /**
+   * @}
+   */
+
   /**
    * The Monitor instance associated with this class.
    */
@@ -110,7 +130,7 @@ class Elector {
    * If we are acked by everyone in the MonMap, we will declare
    * victory.  Also note each peer's feature set.
    */
-  map<int, uint64_t> acked_me;
+  map<int, elector_features_t> acked_me;
   set<int> classic_mons;
   /**
    * @}

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 843b262272dcf8b44f6bd576726af09a423a0c12..2f340f81dc4fb7b4fa96cdd817dde2513fb067ef 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -790,7 +790,6 @@ int Monitor::init()
   // i'm ready!
   messenger->add_dispatcher_tail(this);
 
-
   bootstrap();
 
   // encode command sets
@@ -1849,7 +1848,10 @@ void Monitor::win_standalone_election()
   const MonCommand *my_cmds;
   int cmdsize;
   get_locally_supported_monitor_commands(&my_cmds, &cmdsize);
-  win_election(elector.get_epoch(), q, CEPH_FEATURES_ALL, my_cmds, cmdsize, NULL);
+  win_election(elector.get_epoch(), q,
+               CEPH_FEATURES_ALL,
+               ceph::features::mon::get_supported(),
+               my_cmds, cmdsize, NULL);
 }
 
 const utime_t& Monitor::get_leader_since() const
@@ -1876,17 +1878,21 @@ void Monitor::_finish_svc_election()
 }
 
 void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
+                           const mon_feature_t& mon_features,
                            const MonCommand *cmdset, int cmdsize,
                            const set<int> *classic_monitors)
 {
   dout(10) << __func__ << " epoch " << epoch << " quorum " << active
-          << " features " << features << dendl;
+          << " features " << features
+           << " mon_features " << mon_features
+           << dendl;
   assert(is_electing());
   state = STATE_LEADER;
   leader_since = ceph_clock_now(g_ceph_context);
   leader = rank;
   quorum = active;
   quorum_features = features;
+  quorum_mon_features = mon_features;
   outside_quorum.clear();
 
   clog->info() << "mon." << name << "@" << rank
@@ -1923,7 +1929,9 @@ void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
   update_mon_metadata(rank, std::move(my_meta));
 }
 
-void Monitor::lose_election(epoch_t epoch, set<int> &q, int l, uint64_t features) 
+void Monitor::lose_election(epoch_t epoch, set<int> &q, int l,
+                            uint64_t features,
+                            const mon_feature_t& mon_features)
 {
   state = STATE_PEON;
   leader_since = utime_t();
@@ -1931,8 +1939,11 @@ void Monitor::lose_election(epoch_t epoch, set<int> &q, int l, uint64_t features
   quorum = q;
   outside_quorum.clear();
   quorum_features = features;
+  quorum_mon_features = mon_features;
   dout(10) << "lose_election, epoch " << epoch << " leader is mon" << leader
-          << " quorum is " << quorum << " features are " << quorum_features << dendl;
+          << " quorum is " << quorum << " features are " << quorum_features
+           << " mon_features are " << quorum_mon_features
+           << dendl;
 
   paxos->peon_init();
   _finish_svc_election();

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 29f97d60b5077170167612289b66ec7c4c23b6eb..bfdc5e59e3026cd7c04caf106124fab07931675d 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -216,6 +216,10 @@ private:
   utime_t leader_since;  // when this monitor became the leader, if it is the leader
   utime_t exited_quorum; // time detected as not in quorum; 0 if in
   uint64_t quorum_features;  ///< intersection of quorum member feature bits
+  /**
+   * Intersection of quorum members mon-specific feature bits
+   */
+  mon_feature_t quorum_mon_features;
   bufferlist supported_commands_bl; // encoded MonCommands we support
   bufferlist classic_commands_bl; // encoded MonCommands supported by Dumpling
   set<int> classic_mons; // set of "classic" monitors; only valid on leader
@@ -601,10 +605,13 @@ public:
   // end election (called by Elector)
   void win_election(epoch_t epoch, set<int>& q,
                    uint64_t features,
+                    const mon_feature_t& mon_features,
                    const MonCommand *cmdset, int cmdsize,
                    const set<int> *classic_monitors);
   void lose_election(epoch_t epoch, set<int>& q, int l,
-                    uint64_t features); // end election (called by Elector)
+                    uint64_t features,
+                     const mon_feature_t& mon_features);
+  // end election (called by Elector)
   void finish_election();
 
   const bufferlist& get_supported_commands_bl() {