From dd1a25218c5a1def02146edb2dce0d97a71e4436 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Wed, 9 Aug 2017 12:50:57 -0400
Subject: [PATCH] osd/PG: force rebuild of missing set on jewel upgrade

Previously we were detecting the need to rebuild missing based on
whether the "divergent_priors" omap key was present.  Unfortunately,
jewel does not always set this, so it is not a reliable indicator.
(It only gets set if you actually have a divergent prior at some
point in the PG's life time on that OSD.)

Fix by using the info_struct_v on the PG to detect whether we need
to do the conversion.  We didn't bump the value when we adding
the missing persistence, but the fastinfo was also added during
the same period between jewel and kraken, so it will work just as
well.

Fixes: http://tracker.ceph.com/issues/20958
Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/osd/PG.cc                      | 18 ++++++++++++++++++
 src/osd/PGLog.h                    | 13 +++++++------
 src/tools/ceph_objectstore_tool.cc |  4 +++-
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index a0d9fd65979..e0707efe2d5 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3315,6 +3315,17 @@ void PG::read_state(ObjectStore *store, bufferlist &bl)
 
   last_written_info = info;
 
+  // if we are upgrading from jewel, we need to force rebuild of
+  // missing set.  v9 was fastinfo, added v11.0.2-331-g1d5dc29a13
+  // (before kraken).  persisted missing set was circa
+  // v11.0.0-866-gb0e239da95 (a bit earlier, also before kraken).
+  // v8 was pre-jewel (per-pg meta object).
+  bool force_rebuild_missing = info_struct_v < 9;
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " detected upgrade from jewel, force_rebuild_missing"
+	     << dendl;
+  }
+
   ostringstream oss;
   pg_log.read_log_and_missing(
     store,
@@ -3322,12 +3333,19 @@ void PG::read_state(ObjectStore *store, bufferlist &bl)
     info_struct_v < 8 ? coll_t::meta() : coll,
     ghobject_t(info_struct_v < 8 ? OSD::make_pg_log_oid(pg_id) : pgmeta_oid),
     info,
+    force_rebuild_missing,
     oss,
     cct->_conf->osd_ignore_stale_divergent_priors,
     cct->_conf->osd_debug_verify_missing_on_start);
   if (oss.tellp())
     osd->clog->error() << oss.rdbuf();
 
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " forced rebuild of missing got "
+	     << pg_log.get_missing()
+	     << dendl;
+  }
+
   // log any weirdness
   log_weirdness();
 }
diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h
index 1ddabbc40ec..c86bfb5bd13 100644
--- a/src/osd/PGLog.h
+++ b/src/osd/PGLog.h
@@ -1238,13 +1238,14 @@ public:
     coll_t log_coll,
     ghobject_t log_oid,
     const pg_info_t &info,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool debug_verify_stored_missing = false
     ) {
     return read_log_and_missing(
       store, pg_coll, log_coll, log_oid, info,
-      log, missing, oss,
+      log, missing, force_rebuild_missing, oss,
       tolerate_divergent_missing_log,
       &clear_divergent_priors,
       this,
@@ -1261,6 +1262,7 @@ public:
     const pg_info_t &info,
     IndexedLog &log,
     missing_type &missing,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool *clear_divergent_priors = nullptr,
@@ -1282,7 +1284,6 @@ public:
     eversion_t on_disk_rollback_info_trimmed_to = eversion_t();
     ObjectMap::ObjectMapIterator p = store->get_omap_iterator(log_coll, log_oid);
     map<eversion_t, hobject_t> divergent_priors;
-    bool has_divergent_priors = false;
     missing.may_include_deletes = false;
     list<pg_log_entry_t> entries;
     list<pg_log_dup_t> dups;
@@ -1297,7 +1298,7 @@ public:
 	  ::decode(divergent_priors, bp);
 	  ldpp_dout(dpp, 20) << "read_log_and_missing " << divergent_priors.size()
 			     << " divergent_priors" << dendl;
-	  has_divergent_priors = true;
+	  assert(force_rebuild_missing);
 	  debug_verify_stored_missing = false;
 	} else if (p->key() == "can_rollback_to") {
 	  ::decode(on_disk_can_rollback_to, bp);
@@ -1344,7 +1345,7 @@ public:
       std::move(entries),
       std::move(dups));
 
-    if (has_divergent_priors || debug_verify_stored_missing) {
+    if (force_rebuild_missing || debug_verify_stored_missing) {
       // build missing
       if (debug_verify_stored_missing || info.last_complete < info.last_update) {
 	ldpp_dout(dpp, 10)
@@ -1437,7 +1438,7 @@ public:
 	    }
 	  }
 	} else {
-	  assert(has_divergent_priors);
+	  assert(force_rebuild_missing);
 	  for (map<eversion_t, hobject_t>::reverse_iterator i =
 		 divergent_priors.rbegin();
 	       i != divergent_priors.rend();
@@ -1491,7 +1492,7 @@ public:
       }
     }
 
-    if (!has_divergent_priors) {
+    if (!force_rebuild_missing) {
       if (clear_divergent_priors)
 	(*clear_divergent_priors) = false;
       missing.flush();
diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index d17a74dbcbb..5853f6a0460 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -328,7 +328,9 @@ int get_log(ObjectStore *fs, __u8 struct_ver,
     PGLog::read_log_and_missing(fs, coll,
 		    struct_ver >= 8 ? coll : coll_t::meta(),
 		    struct_ver >= 8 ? pgid.make_pgmeta_oid() : log_oid,
-		    info, log, missing, oss,
+		    info, log, missing,
+				struct_ver < 9,
+				oss,
 		    g_ceph_context->_conf->osd_ignore_stale_divergent_priors);
     if (debug && oss.str().size())
       cerr << oss.str() << std::endl;
-- 
2.39.5