mon/PGMonitor: clear PGMap data when require_luminous is set

Once the OSDMap flag is set there is no going back. Zero out the on-disk
PGMap data, and clear the in-memory PGMap to free up memory and make
bugs easier to spot.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index 042662d19ad3b8d1bdc6a86e04cf2387fcc0ae74..b815bb89f19957422d9489f52f15377f239dc124 100644 (file)
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -171,6 +171,16 @@ void PGMonitor::create_initial()
 
 void PGMonitor::update_from_paxos(bool *need_bootstrap)
 {
+  if (did_delete)
+    return;
+
+  if (get_value("deleted")) {
+    did_delete = true;
+    dout(10) << __func__ << " deleted, clearing in-memory PGMap" << dendl;
+    pg_map = PGMap();
+    return;
+  }
+
   version_t version = get_last_committed();
   if (version == pg_map.version)
     return;
@@ -230,6 +240,8 @@ void PGMonitor::upgrade_format()
 
 void PGMonitor::post_paxos_update()
 {
+  if (did_delete)
+    return;
   dout(10) << __func__ << dendl;
   OSDMap& osdmap = mon->osdmon()->osdmap;
   if (mon->monmap->get_required_features().contains_all(
@@ -249,6 +261,8 @@ void PGMonitor::handle_osd_timeouts()
 {
   if (!mon->is_leader())
     return;
+  if (did_delete)
+    return;
 
   utime_t now(ceph_clock_now());
   utime_t timeo(g_conf->mon_osd_report_timeout, 0);
@@ -263,6 +277,7 @@ void PGMonitor::handle_osd_timeouts()
 
 void PGMonitor::create_pending()
 {
+  do_delete = false;
   pending_inc = PGMap::Incremental();
   pending_inc.version = pg_map.version + 1;
   if (pg_map.version == 0) {
@@ -428,6 +443,26 @@ void PGMonitor::apply_pgmap_delta(bufferlist& bl)
 
 void PGMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 {
+  string prefix = pgmap_meta_prefix;
+  if (do_delete) {
+    dout(1) << __func__ << " clearing pgmap data at v" << pending_inc.version
+           << dendl;
+    do_delete = false;
+    for (auto key : { "version", "stamp", "last_osdmap_epoch",
+         "last_pg_scan", "full_ratio", "nearfull_ratio" }) {
+      t->erase(prefix, key);
+    }
+    for (auto& p : pg_map.pg_stat) {
+      t->erase(prefix, stringify(p.first));
+    }
+    for (auto& p : pg_map.osd_stat) {
+      t->erase(prefix, stringify(p.first));
+    }
+    put_last_committed(t, pending_inc.version);
+    put_value(t, "deleted", 1);
+    return;
+  }
+
   version_t version = pending_inc.version;
   dout(10) << __func__ << " v " << version << dendl;
   assert(get_last_committed() + 1 == version);
@@ -435,8 +470,6 @@ void PGMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 
   uint64_t features = mon->get_quorum_con_features();
 
-  string prefix = pgmap_meta_prefix;
-
   t->put(prefix, "version", pending_inc.version);
   {
     bufferlist bl;
@@ -851,6 +884,9 @@ void PGMonitor::check_osd_map(epoch_t epoch)
   if (mon->is_peon())
     return;  // whatever.
 
+  if (did_delete)
+    return;
+
   if (pg_map.last_osdmap_epoch >= epoch) {
     dout(10) << __func__ << " already seen " << pg_map.last_osdmap_epoch
              << " >= " << epoch << dendl;
@@ -869,11 +905,19 @@ void PGMonitor::check_osd_map(epoch_t epoch)
     return;
   }
 
+  const OSDMap& osdmap = mon->osdmon()->osdmap;
+  if (!did_delete && osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+    // delete all my data
+    dout(1) << __func__ << " will clear pg_map data" << dendl;
+    do_delete = true;
+    propose_pending();
+    return;
+  }
+
   // osds that went up or down
   set<int> need_check_down_pg_osds;
 
   // apply latest map(s)
-  const OSDMap& osdmap = mon->osdmon()->osdmap;
   epoch = std::max(epoch, osdmap.get_epoch());
   for (epoch_t e = pg_map.last_osdmap_epoch+1;
        e <= epoch;

diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h
index 6ba82ea12b5857a5430e074bf589c8337ce76460..654441591dca08e50521582f4baefb1cba89f0e7 100644 (file)
--- a/src/mon/PGMonitor.h
+++ b/src/mon/PGMonitor.h
@@ -39,6 +39,9 @@ class PGMonitor : public PaxosService {
   PGMap pg_map;
   std::unique_ptr<PGStatService> pgservice;
 
+  bool do_delete = false;   ///< propose deleting pgmap data
+  bool did_delete = false;  ///< we already deleted pgmap data
+
 private:
   PGMap::Incremental pending_inc;
 

diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h
index df32c352159ad649aa80e02f57f919e286f695ca..050270586f0d6cb2cabb47e98a241f488d258e47 100644 (file)
--- a/src/mon/PaxosService.h
+++ b/src/mon/PaxosService.h
@@ -774,6 +774,18 @@ public:
     t->put(get_service_name(), key, bl);
   }
 
+  /**
+   * Put integer value @v into the key @p key.
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param key The key to which we will add the value
+   * @param v An integer
+   */
+  void put_value(MonitorDBStore::TransactionRef t,
+                const string& key, version_t v) {
+    t->put(get_service_name(), key, v);
+  }
+
   /**
    * @}
    */