osd/PG: force rebuild of missing set on jewel upgrade

Previously we were detecting the need to rebuild missing based on
whether the "divergent_priors" omap key was present.  Unfortunately,
jewel does not always set this, so it is not a reliable indicator.
(It only gets set if you actually have a divergent prior at some
point in the PG's life time on that OSD.)

Fix by using the info_struct_v on the PG to detect whether we need
to do the conversion.  We didn't bump the value when we adding
the missing persistence, but the fastinfo was also added during
the same period between jewel and kraken, so it will work just as
well.

Fixes: http://tracker.ceph.com/issues/20958
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index a0d9fd659796f7bc29fff461e634e093eebf764d..e0707efe2d581bf8ce1ca81ff5c3ab2391e8c01c 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -3315,6 +3315,17 @@ void PG::read_state(ObjectStore *store, bufferlist &bl)
 
   last_written_info = info;
 
+  // if we are upgrading from jewel, we need to force rebuild of
+  // missing set.  v9 was fastinfo, added v11.0.2-331-g1d5dc29a13
+  // (before kraken).  persisted missing set was circa
+  // v11.0.0-866-gb0e239da95 (a bit earlier, also before kraken).
+  // v8 was pre-jewel (per-pg meta object).
+  bool force_rebuild_missing = info_struct_v < 9;
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " detected upgrade from jewel, force_rebuild_missing"
+            << dendl;
+  }
+
   ostringstream oss;
   pg_log.read_log_and_missing(
     store,
@@ -3322,12 +3333,19 @@ void PG::read_state(ObjectStore *store, bufferlist &bl)
     info_struct_v < 8 ? coll_t::meta() : coll,
     ghobject_t(info_struct_v < 8 ? OSD::make_pg_log_oid(pg_id) : pgmeta_oid),
     info,
+    force_rebuild_missing,
     oss,
     cct->_conf->osd_ignore_stale_divergent_priors,
     cct->_conf->osd_debug_verify_missing_on_start);
   if (oss.tellp())
     osd->clog->error() << oss.rdbuf();
 
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " forced rebuild of missing got "
+            << pg_log.get_missing()
+            << dendl;
+  }
+
   // log any weirdness
   log_weirdness();
 }

diff --git a/src/osd/PGLog.h b/src/osd/PGLog.h
index 1ddabbc40ecb9b3e313039458cbbe1fadee38c84..c86bfb5bd1364325d09ca34a0c3b560b3b7f2879 100644 (file)
--- a/src/osd/PGLog.h
+++ b/src/osd/PGLog.h
@@ -1238,13 +1238,14 @@ public:
     coll_t log_coll,
     ghobject_t log_oid,
     const pg_info_t &info,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool debug_verify_stored_missing = false
     ) {
     return read_log_and_missing(
       store, pg_coll, log_coll, log_oid, info,
-      log, missing, oss,
+      log, missing, force_rebuild_missing, oss,
       tolerate_divergent_missing_log,
       &clear_divergent_priors,
       this,
@@ -1261,6 +1262,7 @@ public:
     const pg_info_t &info,
     IndexedLog &log,
     missing_type &missing,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool *clear_divergent_priors = nullptr,
@@ -1282,7 +1284,6 @@ public:
     eversion_t on_disk_rollback_info_trimmed_to = eversion_t();
     ObjectMap::ObjectMapIterator p = store->get_omap_iterator(log_coll, log_oid);
     map<eversion_t, hobject_t> divergent_priors;
-    bool has_divergent_priors = false;
     missing.may_include_deletes = false;
     list<pg_log_entry_t> entries;
     list<pg_log_dup_t> dups;
@@ -1297,7 +1298,7 @@ public:
          ::decode(divergent_priors, bp);
          ldpp_dout(dpp, 20) << "read_log_and_missing " << divergent_priors.size()
                             << " divergent_priors" << dendl;
-         has_divergent_priors = true;
+         assert(force_rebuild_missing);
          debug_verify_stored_missing = false;
        } else if (p->key() == "can_rollback_to") {
          ::decode(on_disk_can_rollback_to, bp);
@@ -1344,7 +1345,7 @@ public:
       std::move(entries),
       std::move(dups));
 
-    if (has_divergent_priors || debug_verify_stored_missing) {
+    if (force_rebuild_missing || debug_verify_stored_missing) {
       // build missing
       if (debug_verify_stored_missing || info.last_complete < info.last_update) {
        ldpp_dout(dpp, 10)
@@ -1437,7 +1438,7 @@ public:
            }
          }
        } else {
-         assert(has_divergent_priors);
+         assert(force_rebuild_missing);
          for (map<eversion_t, hobject_t>::reverse_iterator i =
                 divergent_priors.rbegin();
               i != divergent_priors.rend();
@@ -1491,7 +1492,7 @@ public:
       }
     }
 
-    if (!has_divergent_priors) {
+    if (!force_rebuild_missing) {
       if (clear_divergent_priors)
        (*clear_divergent_priors) = false;
       missing.flush();

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index d17a74dbcbb0ec6c18864361365a84d75b5099b9..5853f6a04608b022817f71215e839ff5e6640021 100644 (file)
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -328,7 +328,9 @@ int get_log(ObjectStore *fs, __u8 struct_ver,
     PGLog::read_log_and_missing(fs, coll,
                    struct_ver >= 8 ? coll : coll_t::meta(),
                    struct_ver >= 8 ? pgid.make_pgmeta_oid() : log_oid,
-                   info, log, missing, oss,
+                   info, log, missing,
+                               struct_ver < 9,
+                               oss,
                    g_ceph_context->_conf->osd_ignore_stale_divergent_priors);
     if (debug && oss.str().size())
       cerr << oss.str() << std::endl;