osd: request new map from PG when needed

The fast dispatch refactor in 3cc48278bf0ee5c9535d04b60a661f988c50063b
eliminated the osdmap subscription in the ms_fast_dispatch path, which
meant ops could reach a PG without having the latest map. In a cluster
with few osdmap updates, where the monitor fails to send a new map to
an osd (it tries one random osd), this can result in indefinitely
blocked requests.

Fix this by adding an OSDService mechanism for scheduling a new osdmap
subscription request.

Fixes: http://tracker.ceph.com/issues/21428
Signed-off-by: Josh Durgin <jdurgin@redhat.com>
(cherry picked from commit dd33360e5a4ed06228af552edadee88e66c5aba9)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 4ee55e32d0ef6ce266e61f929e5a724640adc772..426da2440089115065c5c2e191580b1137636ee0 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -585,6 +585,11 @@ void OSDService::activate_map()
   agent_lock.Unlock();
 }
 
+void OSDService::request_osdmap_update(epoch_t e)
+{
+  osd->osdmap_subscribe(e, false);
+}
+
 class AgentTimeoutCB : public Context {
   PGRef pg;
 public:
@@ -1913,6 +1918,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   disk_tp(cct, "OSD::disk_tp", "tp_osd_disk", cct->_conf->osd_disk_threads, "osd_disk_threads"),
   command_tp(cct, "OSD::command_tp", "tp_osd_cmd",  1),
   session_waiting_lock("OSD::session_waiting_lock"),
+  osdmap_subscribe_lock("OSD::osdmap_subscribe_lock"),
   heartbeat_lock("OSD::heartbeat_lock"),
   heartbeat_stop(false),
   heartbeat_need_update(true),
@@ -7489,10 +7495,12 @@ struct C_OnMapApply : public Context {
 
 void OSD::osdmap_subscribe(version_t epoch, bool force_request)
 {
-  OSDMapRef osdmap = service.get_osdmap();
-  if (osdmap->get_epoch() >= epoch)
+  Mutex::Locker l(osdmap_subscribe_lock);
+  if (latest_subscribed_epoch >= epoch && !force_request)
     return;
 
+  latest_subscribed_epoch = MAX(epoch, latest_subscribed_epoch);
+
   if (monc->sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
       force_request) {
     monc->renew_subs();

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 88e84710ff531bf4f9eee2a59d4df2762c469a97..42a152b3c9adc6aca9fc27ef89cab5ee20edde84 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1144,6 +1144,8 @@ public:
     return ret;
   }
 
+  void request_osdmap_update(epoch_t e);
+
   // -- stopping --
   Mutex is_stopping_lock;
   Cond is_stopping_cond;
@@ -1455,6 +1457,9 @@ private:
   void osdmap_subscribe(version_t epoch, bool force_request);
   /** @} monc helpers */
 
+  Mutex osdmap_subscribe_lock;
+  epoch_t latest_subscribed_epoch{0};
+
   // -- heartbeat --
   /// information about a heartbeat peer
   struct HeartbeatInfo {

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index f1ef2e57cd0e8b72b02ec90418f173fa02455155..7a6c24a0c2800e836d15f54d5d8f8a8589e8021b 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1648,6 +1648,7 @@ void PrimaryLogPG::do_request(
             << p->first << " not empty, queueing" << dendl;
     p->second.push_back(op);
     op->mark_delayed("waiting_for_map not empty");
+    osd->request_osdmap_update(op->min_epoch);
     return;
   }
   if (!have_same_or_newer_map(op->min_epoch)) {