osd: request new map from PG when needed

The fast dispatch refactor in 3cc48278bf0ee5c9535d04b60a661f988c50063b
eliminated the osdmap subscription in the ms_fast_dispatch path, which
meant ops could reach a PG without having the latest map. In a cluster
with few osdmap updates, where the monitor fails to send a new map to
an osd (it tries one random osd), this can result in indefinitely
blocked requests.

Fix this by adding an OSDService mechanism for scheduling a new osdmap
subscription request.

Fixes: http://tracker.ceph.com/issues/21428
Signed-off-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index cec9eb229efc21146e2b2a9396e54b9d0a6b43a6..40689e757ead8444a6b45571800ac002d796802c 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -591,6 +591,11 @@ void OSDService::activate_map()
   agent_lock.Unlock();
 }
 
+void OSDService::request_osdmap_update(epoch_t e)
+{
+  osd->osdmap_subscribe(e, false);
+}
+
 class AgentTimeoutCB : public Context {
   PGRef pg;
 public:
@@ -1919,6 +1924,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   disk_tp(cct, "OSD::disk_tp", "tp_osd_disk", cct->_conf->osd_disk_threads, "osd_disk_threads"),
   command_tp(cct, "OSD::command_tp", "tp_osd_cmd",  1),
   session_waiting_lock("OSD::session_waiting_lock"),
+  osdmap_subscribe_lock("OSD::osdmap_subscribe_lock"),
   heartbeat_lock("OSD::heartbeat_lock"),
   heartbeat_stop(false),
   heartbeat_need_update(true),
@@ -7253,10 +7259,12 @@ struct C_OnMapApply : public Context {
 
 void OSD::osdmap_subscribe(version_t epoch, bool force_request)
 {
-  OSDMapRef osdmap = service.get_osdmap();
-  if (osdmap->get_epoch() >= epoch)
+  Mutex::Locker l(osdmap_subscribe_lock);
+  if (latest_subscribed_epoch >= epoch && !force_request)
     return;
 
+  latest_subscribed_epoch = MAX(epoch, latest_subscribed_epoch);
+
   if (monc->sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
       force_request) {
     monc->renew_subs();

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 52692f7057ce403af27e289dbcf21d6f10dd0af2..3c3c36bd21240c37da45818e4a559e510cd84546 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1141,6 +1141,8 @@ public:
     return ret;
   }
 
+  void request_osdmap_update(epoch_t e);
+
   // -- stopping --
   Mutex is_stopping_lock;
   Cond is_stopping_cond;
@@ -1451,6 +1453,9 @@ private:
   void osdmap_subscribe(version_t epoch, bool force_request);
   /** @} monc helpers */
 
+  Mutex osdmap_subscribe_lock;
+  epoch_t latest_subscribed_epoch{0};
+
   // -- heartbeat --
   /// information about a heartbeat peer
   struct HeartbeatInfo {

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 76ea8ac40076bf7be40eab6ce5776fd3fdca8de8..de0ea8df453771e03f4dcc14c64af3af3a8afac7 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1646,6 +1646,7 @@ void PrimaryLogPG::do_request(
             << p->first << " not empty, queueing" << dendl;
     p->second.push_back(op);
     op->mark_delayed("waiting_for_map not empty");
+    osd->request_osdmap_update(op->min_epoch);
     return;
   }
   if (!have_same_or_newer_map(op->min_epoch)) {