osd: Add automatic repair for DBObjectMap bug

Add repair command to ceph-osdomap-tool too

Under some situations the previous rm_keys() code would
generated a corrupt complete table.  There is no way to
figure out what the table should look like now.  By removing
the entries we fix the corruption and aren't much worse off
because the corruption caused some deleted keys to re-appear.

This doesn't breaking the parent/child relationship during
repair because some of the keys may still be contained
in the parent.

Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/os/ObjectMap.h b/src/os/ObjectMap.h
index 05361820af3195a6c394525ca68a88342ab49ae0..a69b70148f780179c7b097ca8c5d33a6a826830e 100644 (file)
--- a/src/os/ObjectMap.h
+++ b/src/os/ObjectMap.h
@@ -138,7 +138,7 @@ public:
     const SequencerPosition *spos=0   ///< [in] Sequencer
     ) { return 0; }
 
-  virtual int check(std::ostream &out) { return 0; }
+  virtual int check(std::ostream &out, bool repair = false) { return 0; }
 
   typedef KeyValueDB::GenericIteratorImpl ObjectMapIteratorImpl;
   typedef ceph::shared_ptr<ObjectMapIteratorImpl> ObjectMapIterator;

diff --git a/src/os/filestore/DBObjectMap.cc b/src/os/filestore/DBObjectMap.cc
index 1833631d20581a9df878740513ed08487571c0be..37a95fee3575720c0b73e8a1b64db5aafcb27877 100644 (file)
--- a/src/os/filestore/DBObjectMap.cc
+++ b/src/os/filestore/DBObjectMap.cc
@@ -55,7 +55,7 @@ static void append_escaped(const string &in, string *out)
   }
 }
 
-int DBObjectMap::check(std::ostream &out)
+int DBObjectMap::check(std::ostream &out, bool repair)
 {
   int errors = 0;
   map<uint64_t, uint64_t> parent_to_num_children;
@@ -78,18 +78,25 @@ int DBObjectMap::check(std::ostream &out)
            complete_iter->next()) {
          if (prev && prev >= complete_iter->key()) {
              out << "Bad complete for " << header.oid << std::endl;
-             errors++;
              complete_error = true;
              break;
          }
          prev = string(complete_iter->value().c_str(), complete_iter->value().length() - 1);
       }
       if (complete_error) {
-        out << "Complete mapping:" << std::endl;
+        out << "Complete mapping for " << header.seq << " :" << std::endl;
         for (complete_iter->seek_to_first(); complete_iter->valid();
              complete_iter->next()) {
           out << complete_iter->key() << " -> " << string(complete_iter->value().c_str(), complete_iter->value().length() - 1) << std::endl;
         }
+        if (repair) {
+          KeyValueDB::Transaction t = db->get_transaction();
+          t->rmkeys_by_prefix(USER_PREFIX + header_key(header.seq) + COMPLETE_PREFIX);
+          db->submit_transaction(t);
+          out << "Cleared complete mapping to repair" << std::endl;
+        } else {
+          errors++;  // Only count when not repaired
+        }
       }
 
       if (header.parent == 0)
@@ -1051,6 +1058,12 @@ int DBObjectMap::init(bool do_upgrade)
     state.v = 2;
     state.seq = 1;
   }
+  ostringstream ss;
+  int errors = check(ss, true);
+  if (errors) {
+    dout(5) << ss.str() << dendl;
+    return -EINVAL;
+  }
   dout(20) << "(init)dbobjectmap: seq is " << state.seq << dendl;
   return 0;
 }

diff --git a/src/os/filestore/DBObjectMap.h b/src/os/filestore/DBObjectMap.h
index cf1043fe2f722ee06512f9ca9f637d6d592c4958..40108e711d56e80328aead9c67e62c08350f6e77 100644 (file)
--- a/src/os/filestore/DBObjectMap.h
+++ b/src/os/filestore/DBObjectMap.h
@@ -213,7 +213,7 @@ public:
   int upgrade_to_v2();
 
   /// Consistency check, debug, there must be no parallel writes
-  int check(std::ostream &out) override;
+  int check(std::ostream &out, bool repair = false) override;
 
   /// Ensure that all previous operations are durable
   int sync(const ghobject_t *oid=0, const SequencerPosition *spos=0) override;

diff --git a/src/tools/ceph_osdomap_tool.cc b/src/tools/ceph_osdomap_tool.cc
index 2be4ee58de4c69bc8a0672c7e7731bad5ba39d67..1e18c237d09223d4de526a35197fe18f1299a3cd 100644 (file)
--- a/src/tools/ceph_osdomap_tool.cc
+++ b/src/tools/ceph_osdomap_tool.cc
@@ -37,7 +37,7 @@ int main(int argc, char **argv) {
     ("debug", "Additional debug output from DBObjectMap")
     ("oid", po::value<string>(&oid), "Restrict to this object id when dumping objects")
     ("command", po::value<string>(&cmd),
-     "command arg is one of [dump-raw-keys, dump-raw-key-vals, dump-objects, dump-objects-with-keys, check, dump-headers], mandatory")
+     "command arg is one of [dump-raw-keys, dump-raw-key-vals, dump-objects, dump-objects-with-keys, check, dump-headers, repair], mandatory")
     ;
   po::positional_options_description p;
   p.add("command", 1);
@@ -109,6 +109,9 @@ int main(int argc, char **argv) {
     std::cerr << "Output: " << out.str() << std::endl;
     goto done;
   }
+  // We don't call omap.init() here because it will repair
+  // the DBObjectMap which we might want to examine for diagnostic
+  // reasons.  Instead use --command repair.
   r = 0;
 
 
@@ -157,14 +160,15 @@ int main(int argc, char **argv) {
        j->value().hexdump(std::cout);
       }
     }
-  } else if (cmd == "check") {
-    r = omap.check(std::cout);
+  } else if (cmd == "check" || cmd == "repair") {
+    bool repair = (cmd == "repair");
+    r = omap.check(std::cout, repair);
     if (r > 0) {
       std::cerr << "check got " << r << " error(s)" << std::endl;
       r = 1;
       goto done;
     }
-    std::cout << "check succeeded" << std::endl;
+    std::cout << (repair ? "repair" : "check") << " succeeded" << std::endl;
   } else if (cmd == "dump-headers") {
     vector<DBObjectMap::_Header> headers;
     r = omap.list_object_headers(&headers);