osd: remove all stale osdmaps in handle_osd_map()

in a large cluster, there are better chances that the OSD fails to trim
the cached osdmap in a timely manner. and sometimes, it is just unable
to keep up with the incoming osdmap if skip_maps, so the osdmap cache
can keep building up to over 250GB in size. in this change

* publish_superblock() before trimming the osdmaps, so other osdmap
  consumers of OSDService.superblock won't access the osdmaps being
  removed.
* trim all stale osdmaps in batch of conf->osd_target_transaction_size
  if skip_maps is true. in my test, it happens when the osd only
  receives the osdmap from monitor occasionally because the osd happens
  to be chosen when monitor wants to share a new osdmap with a random
  osd.
* always use dedicated transaction(s) for trimming osdmaps. so even in
  the normal case where we are able to trim all stale osdmaps in a
  single batch, a separated transaction is used. we can piggy back
  the commits for removing maps, but we keep it this way for simplicity.
* use std::min() instead MIN() for type safety

Fixes: http://tracker.ceph.com/issues/13990
Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 7caef4ef946e1a59250d8c58ca7ec015a5fe4924..99d7ced1707447bd538a2c93b0b50183ca73529c 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -6530,6 +6530,44 @@ void OSD::osdmap_subscribe(version_t epoch, bool force_request)
   }
 }
 
+void OSD::trim_maps(epoch_t oldest, int nreceived, bool skip_maps)
+{
+  epoch_t min = std::min(oldest, service.map_cache.cached_key_lower_bound());
+  if (min <= superblock.oldest_map)
+    return;
+
+  int num = 0;
+  ObjectStore::Transaction t;
+  for (epoch_t e = superblock.oldest_map; e < min; ++e) {
+    dout(20) << " removing old osdmap epoch " << e << dendl;
+    t.remove(coll_t::meta(), get_osdmap_pobject_name(e));
+    t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(e));
+    superblock.oldest_map = e + 1;
+    num++;
+    if (num >= cct->_conf->osd_target_transaction_size && num >= nreceived) {
+      service.publish_superblock(superblock);
+      write_superblock(t);
+      store->queue_transaction(service.meta_osr.get(), std::move(t), nullptr);
+      num = 0;
+      if (!skip_maps) {
+       // skip_maps leaves us with a range of old maps if we fail to remove all
+       // of them before moving superblock.oldest_map forward to the first map
+       // in the incoming MOSDMap msg. so we should continue removing them in
+       // this case, even we could do huge series of delete transactions all at
+       // once.
+       break;
+      }
+    }
+  }
+  if (num > 0) {
+    service.publish_superblock(superblock);
+    write_superblock(t);
+    store->queue_transaction(service.meta_osr.get(), std::move(t), nullptr);
+  }
+  // we should not remove the cached maps
+  assert(min <= service.map_cache.cached_key_lower_bound());
+}
+
 void OSD::handle_osd_map(MOSDMap *m)
 {
   assert(osd_lock.is_locked());
@@ -6705,23 +6743,8 @@ void OSD::handle_osd_map(MOSDMap *m)
   }
 
   if (superblock.oldest_map) {
-    int num = 0;
-    epoch_t cache_lb = service.map_cache.cached_key_lower_bound();
-    epoch_t min = MIN(m->oldest_map, cache_lb);
-    dout(20) << __func__ << " oldest_map " << m->oldest_map
-            << " cache lb " << cache_lb
-            << " min " << min << dendl;
-    for (epoch_t e = superblock.oldest_map; e < min; ++e) {
-      dout(20) << " removing old osdmap epoch " << e << dendl;
-      t.remove(coll_t::meta(), get_osdmap_pobject_name(e));
-      t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(e));
-      superblock.oldest_map = e+1;
-      num++;
-      // make sure we at least keep pace with incoming maps
-      if (num >= cct->_conf->osd_target_transaction_size &&
-         (uint64_t)num > (last - first))
-       break;
-    }
+    // make sure we at least keep pace with incoming maps
+    trim_maps(m->oldest_map, last - first + 1, skip_maps);
   }
 
   if (!superblock.oldest_map || skip_maps)

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index cdf2afd45dc231d3aa1bb1da98956d05d282df42..d4abf7de03f665bcfa80ea85daa6d377caef6319 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1878,6 +1878,7 @@ private:
   void wait_for_new_map(OpRequestRef op);
   void handle_osd_map(class MOSDMap *m);
   void _committed_osd_maps(epoch_t first, epoch_t last, class MOSDMap *m);
+  void trim_maps(epoch_t oldest, int nreceived, bool skip_maps);
   void note_down_osd(int osd);
   void note_up_osd(int osd);
   friend class C_OnMapCommit;