osd: populate the trim_thru epoch using MOSDMap.oldest_map

instead of filling MOSDMap with the local oldest_map, we share
the maximum MOSDMap.oldest_map received so far with peers. That
way one OSD's failure to trim won't prevent it from sharing with
others that they are allowed to trim.

Fixes: #13990
Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 3311d3f8d444629c9d7a43e736f961f2865fdd20..673454b3286c5c9f78b48a7f84f37b51547fbff0 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -223,6 +223,7 @@ OSDService::OSDService(OSD *osd) :
   pg_epoch_lock("OSDService::pg_epoch_lock"),
   publish_lock("OSDService::publish_lock"),
   pre_publish_lock("OSDService::pre_publish_lock"),
+  max_oldest_map(0),
   peer_map_epoch_lock("OSDService::peer_map_epoch_lock"),
   sched_scrub_lock("OSDService::sched_scrub_lock"), scrubs_pending(0),
   scrubs_active(0),
@@ -1183,7 +1184,7 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
                                                OSDSuperblock& sblock)
 {
   MOSDMap *m = new MOSDMap(monc->get_fsid());
-  m->oldest_map = sblock.oldest_map;
+  m->oldest_map = max_oldest_map;
   m->newest_map = sblock.newest_map;
 
   for (epoch_t e = to; e > since; e--) {
@@ -1223,7 +1224,7 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
     if (since < sblock.oldest_map) {
       // just send latest full map
       MOSDMap *m = new MOSDMap(monc->get_fsid());
-      m->oldest_map = sblock.oldest_map;
+      m->oldest_map = max_oldest_map;
       m->newest_map = sblock.newest_map;
       get_map_bl(to, m->maps[to]);
       send_map(m, con);
@@ -2161,6 +2162,7 @@ int OSD::init()
   service.init();
   service.publish_map(osdmap);
   service.publish_superblock(superblock);
+  service.max_oldest_map = superblock.oldest_map;
 
   osd_lock.Unlock();
 
@@ -6566,6 +6568,10 @@ void OSD::handle_osd_map(MOSDMap *m)
   logger->inc(l_osd_mape, last - first + 1);
   if (first <= osdmap->get_epoch())
     logger->inc(l_osd_mape_dup, osdmap->get_epoch() - first + 1);
+  if (service.max_oldest_map < m->oldest_map) {
+    service.max_oldest_map = m->oldest_map;
+    assert(service.max_oldest_map >= superblock.oldest_map);
+  }
 
   // make sure there is something new, here, before we bother flushing the queues and such
   if (last <= osdmap->get_epoch()) {

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 6cc2f96a3acc1311d089844dab21851da4c34e63..e57991132a22acacff6a8128154856c62e3270d3 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -40,6 +40,7 @@
 #include "messages/MOSDRepScrub.h"
 #include "OpRequest.h"
 
+#include <atomic>
 #include <map>
 #include <memory>
 #include "include/memory.h"
@@ -475,6 +476,7 @@ public:
 
   int get_nodeid() const { return whoami; }
 
+  std::atomic<epoch_t> max_oldest_map;
   OSDMapRef osdmap;
   OSDMapRef get_osdmap() {
     Mutex::Locker l(publish_lock);