osd: reliably send pg_created messages to the mon

The OSD has to reliably deliver a pg_created message to the mon in order
for the mon to clear the pool's CREATING flag.  Previously, a mon
connection reset would drop the message.

Restructure this to:

- queue a message any time a PG peers and the pool as the CREATING flag
- track pending messages in OSDService
- resend on mon connect
- prune messages for pools that no longer have the CREATING flag

This new strategy can result in resends of these messages to the mon in
cases where the mon already knows the PG was created.  However, pool
creation is rare, and these extra messages are cheap.  And we can avoid
this overhead if we like by limiting the number of PGs that the mon can
create explicitly if we choose (by lowering mon_osd_max_initial_pgs).

Fixes: http://tracker.ceph.com/issues/37775
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 287a1ed0b9ca9d0810fa2f2a48f2c1c25fcbda88..c3337d301cc7c3cd0af08f44e7e4acac818d046a 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1028,12 +1028,46 @@ void OSDService::send_pg_temp()
 
 void OSDService::send_pg_created(pg_t pgid)
 {
+  std::lock_guard l(pg_created_lock);
   dout(20) << __func__ << dendl;
-  if (osdmap->require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+  auto o = get_osdmap();
+  if (o->require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+    pg_created.insert(pgid);
     monc->send_mon_message(new MOSDPGCreated(pgid));
   }
 }
 
+void OSDService::send_pg_created()
+{
+  std::lock_guard l(pg_created_lock);
+  dout(20) << __func__ << dendl;
+  auto o = get_osdmap();
+  if (o->require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+    for (auto pgid : pg_created) {
+      monc->send_mon_message(new MOSDPGCreated(pgid));
+    }
+  }
+}
+
+void OSDService::prune_pg_created()
+{
+  std::lock_guard l(pg_created_lock);
+  dout(20) << __func__ << dendl;
+  auto o = get_osdmap();
+  auto i = pg_created.begin();
+  while (i != pg_created.end()) {
+    auto p = o->get_pg_pool(i->pool());
+    if (!p || !p->has_flag(pg_pool_t::FLAG_CREATING)) {
+      dout(20) << __func__ << " pruning " << *i << dendl;
+      i = pg_created.erase(i);
+    } else {
+      dout(20) << __func__ << " keeping " << *i << dendl;
+      ++i;
+    }
+  }
+}
+
+
 // --------------------------------------
 // dispatch
 
@@ -5395,6 +5429,7 @@ void OSD::ms_handle_connect(Connection *con)
       service.clear_sent_ready_to_merge();
       service.send_pg_temp();
       service.send_ready_to_merge();
+      service.send_pg_created();
       requeue_failures();
       send_failures();
 
@@ -8373,6 +8408,8 @@ void OSD::consume_map()
     ceph_assert(merge_pgs.empty());
   }
 
+  service.prune_pg_created();
+
   unsigned pushes_to_free = 0;
   for (auto& shard : shards) {
     shard->consume_map(osdmap, &pushes_to_free);

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 358c3fd1bc630674d6cc48cf1be541846f461e2e..d1c16d6bf332693008b0dbd765a0effda14620f5 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -748,7 +748,11 @@ public:
   void requeue_pg_temp();
   void send_pg_temp();
 
+  ceph::mutex pg_created_lock = ceph::make_mutex("OSDService::pg_created_lock");
+  set<pg_t> pg_created;
   void send_pg_created(pg_t pgid);
+  void prune_pg_created();
+  void send_pg_created();
 
   AsyncReserver<spg_t> snap_reserver;
   void queue_recovery_context(PG *pg, GenContext<ThreadPool::TPHandle&> *c);

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 6ac9b4b9849f5d0318b013e2b5541622f8c407ef..88b84b9423e04c38834678fb09b889a78de3ed6f 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3376,10 +3376,6 @@ void PG::publish_stats_to_osd()
     if ((state & (PG_STATE_ACTIVE|PG_STATE_PEERED)) &&
        !(info.stats.state & (PG_STATE_ACTIVE|PG_STATE_PEERED)))
       info.stats.last_became_peered = now;
-    if (!(state & PG_STATE_CREATING) &&
-       (info.stats.state & PG_STATE_CREATING)) {
-      osd->send_pg_created(get_pgid().pgid);
-    }
     info.stats.state = state;
   }
 
@@ -8477,8 +8473,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const AllReplicasActi
     pg->state_set(PG_STATE_ACTIVE);
   }
 
-  // info.last_epoch_started is set during activate()
-  if (pg->info.history.last_epoch_started == 0) {
+  if (pg->pool.info.has_flag(pg_pool_t::FLAG_CREATING)) {
     pg->osd->send_pg_created(pgid);
   }