osdc: expose new blacklist entries from Objecter

For the MDS which needs to know about new blacklist entries
in order to evict corresponding MDS sessions.

Fixes: http://tracker.ceph.com/issues/17980
Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 0364be92de799e4418212edb0e5cae1a67b5fc4c..6582aec99e4c8c71b612200e57aa2eeaffe2ac56 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -1182,11 +1182,20 @@ void Objecter::handle_osd_map(MOSDMap *m)
                        << dendl;
          OSDMap::Incremental inc(m->incremental_maps[e]);
          osdmap->apply_incremental(inc);
+
+          emit_blacklist_events(inc);
+
          logger->inc(l_osdc_map_inc);
        }
        else if (m->maps.count(e)) {
          ldout(cct, 3) << "handle_osd_map decoding full epoch " << e << dendl;
-         osdmap->decode(m->maps[e]);
+          OSDMap *new_osdmap = new OSDMap();
+          new_osdmap->decode(m->maps[e]);
+
+          emit_blacklist_events(*osdmap, *new_osdmap);
+
+          osdmap = new_osdmap;
+
          logger->inc(l_osdc_map_full);
        }
        else {
@@ -1361,6 +1370,58 @@ void Objecter::handle_osd_map(MOSDMap *m)
   }
 }
 
+void Objecter::enable_blacklist_events()
+{
+  unique_lock wl(rwlock);
+
+  blacklist_events_enabled = true;
+}
+
+void Objecter::consume_blacklist_events(std::set<entity_addr_t> *events)
+{
+  unique_lock wl(rwlock);
+
+  if (events->empty()) {
+    events->swap(blacklist_events);
+  } else {
+    for (const auto &i : blacklist_events) {
+      events->insert(i);
+    }
+    blacklist_events.clear();
+  }
+}
+
+void Objecter::emit_blacklist_events(const OSDMap::Incremental &inc)
+{
+  if (!blacklist_events_enabled) {
+    return;
+  }
+
+  for (const auto &i : inc.new_blacklist) {
+    blacklist_events.insert(i.first);
+  }
+}
+
+void Objecter::emit_blacklist_events(const OSDMap &old_osd_map,
+                                     const OSDMap &new_osd_map)
+{
+  if (!blacklist_events_enabled) {
+    return;
+  }
+
+  std::set<entity_addr_t> old_set;
+  std::set<entity_addr_t> new_set;
+
+  old_osd_map.get_blacklist(&old_set);
+  new_osd_map.get_blacklist(&new_set);
+
+  std::set<entity_addr_t> delta_set;
+  std::set_difference(
+      new_set.begin(), new_set.end(), old_set.begin(), old_set.end(),
+      std::inserter(delta_set, delta_set.begin()));
+  blacklist_events.insert(delta_set.begin(), delta_set.end());
+}
+
 // op pool check
 
 void Objecter::C_Op_Map_Latest::finish(int r)

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 883ef5bbb1d94090d5ba1872df4af1b575d1ade5..03b964f5ae7e4ada25ef1e4df37dce7571dd4bbe 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1193,8 +1193,15 @@ private:
   bool honor_osdmap_full;
   bool osdmap_full_try;
 
+  // If this is true, accumulate a set of blacklisted entities
+  // to be drained by consume_blacklist_events.
+  bool blacklist_events_enabled;
+  std::set<entity_addr_t> blacklist_events;
+
 public:
   void maybe_request_map();
+
+  void enable_blacklist_events();
 private:
 
   void _maybe_request_map();
@@ -1980,6 +1987,7 @@ private:
     osdmap(new OSDMap), initialized(0), last_tid(0), client_inc(-1),
     max_linger_id(0), num_in_flight(0), global_op_flags(0),
     keep_balanced_budget(false), honor_osdmap_full(true), osdmap_full_try(false),
+    blacklist_events_enabled(false),
     last_seen_osdmap_version(0), last_seen_pgmap_version(0),
     logger(NULL), tick_event(0), m_request_state_hook(NULL),
     num_homeless_ops(0),
@@ -2073,6 +2081,17 @@ private:
   void handle_osd_map(class MOSDMap *m);
   void wait_for_osd_map();
 
+  /**
+   * Get list of entities blacklisted since this was last called,
+   * and reset the list.
+   *
+   * Uses a std::set because typical use case is to compare some
+   * other list of clients to see which overlap with the blacklisted
+   * addrs.
+   *
+   */
+  void consume_blacklist_events(std::set<entity_addr_t> *events);
+
   int pool_snap_by_name(int64_t poolid,
                        const char *snap_name,
                        snapid_t *snap) const;
@@ -2081,6 +2100,10 @@ private:
   int pool_snap_list(int64_t poolid, vector<uint64_t> *snaps);
 private:
 
+  void emit_blacklist_events(const OSDMap::Incremental &inc);
+  void emit_blacklist_events(const OSDMap &old_osd_map,
+                             const OSDMap &new_osd_map);
+
   // low-level
   void _op_submit(Op *op, shunique_lock& lc, ceph_tid_t *ptid);
   void _op_submit_with_budget(Op *op, shunique_lock& lc,