From: Sage Weil <sage@redhat.com>
Date: Wed, 29 Nov 2017 19:39:59 +0000 (-0600)
Subject: osd: keep maps pinned in cache that pgs haven't consumed yet
X-Git-Tag: v13.0.2~840^2~3
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e19f41ea9c100ec534b89c0381732781d21b1298;p=ceph.git

osd: keep maps pinned in cache that pgs haven't consumed yet

This ensures that we do not drop maps from the map cache that PGs
are still trying to consume.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index c01b14a353fa..6841a4017deb 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1534,9 +1534,43 @@ void OSDService::pin_map_bl(epoch_t e, bufferlist &bl)
 
 void OSDService::clear_map_bl_cache_pins(epoch_t e)
 {
+  // do not rip cache out from under PGs who are processing maps
+  epoch_t min_pg_epoch = get_min_pg_epoch();
+  if (!min_pg_epoch) {
+    // no pgs, use latest osdmap epoch here
+    min_pg_epoch = osdmap->get_epoch();
+  }
+
   Mutex::Locker l(map_cache_lock);
-  map_bl_inc_cache.clear_pinned(e);
-  map_bl_cache.clear_pinned(e);
+  if (e) {
+    map_cache_pinned_epoch = e;
+  }
+  epoch_t actual;
+  if (map_cache_pinned_epoch > min_pg_epoch) {
+    dout(10) << __func__ << " adjusting pin bound " << map_cache_pinned_epoch
+	     << " -> min_pg_epoch " << min_pg_epoch
+	     << dendl;
+    actual = min_pg_epoch;
+    map_cache_pinned_low = true;
+  } else {
+    actual = map_cache_pinned_epoch;
+    if (map_cache_pinned_low) {
+      dout(10) << __func__ << " dropped min_pg_epoch adjustment, back to "
+	       << map_cache_pinned_epoch << dendl;
+      map_cache_pinned_low = false;
+    }
+  }
+  dout(20) << __func__ << " actual " << actual << dendl;
+  map_bl_inc_cache.clear_pinned(actual);
+  map_bl_cache.clear_pinned(actual);
+}
+
+void OSDService::check_map_bl_cache_pins()
+{
+  if (!map_cache_pinned_low) {
+    return;
+  }
+  clear_map_bl_cache_pins(0);
 }
 
 OSDMapRef OSDService::_add_map(OSDMap *o)
@@ -4927,6 +4961,11 @@ void OSD::tick()
   assert(osd_lock.is_locked());
   dout(10) << "tick" << dendl;
 
+  // make sure map cache pin bound is properly adjusted (it may be set
+  // artificially low if the last maps were injested when pg map processing
+  // was behind).
+  service.check_map_bl_cache_pins();
+
   if (is_active() || is_waiting_for_healthy()) {
     maybe_update_heartbeat_peers();
   }
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 3799c937c072..02b01f6376d4 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -937,6 +937,12 @@ public:
   SimpleLRU<epoch_t, bufferlist> map_bl_cache;
   SimpleLRU<epoch_t, bufferlist> map_bl_inc_cache;
 
+  /// newest map fully consumed by handle_osd_map (i.e., written to disk)
+  epoch_t map_cache_pinned_epoch = 0;
+
+  /// true if pg consumption affected our unpinning
+  std::atomic<bool> map_cache_pinned_low = {false};
+
   OSDMapRef try_get_map(epoch_t e);
   OSDMapRef get_map(epoch_t e) {
     OSDMapRef ret(try_get_map(e));
@@ -970,6 +976,7 @@ public:
   bool get_inc_map_bl(epoch_t e, bufferlist& bl);
 
   void clear_map_bl_cache_pins(epoch_t e);
+  void check_map_bl_cache_pins();
 
   void need_heartbeat_peer_update();