From b4727229402af1206109ceb7b5e8e19209ccf316 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Thu, 10 Jun 2010 22:13:22 -0700
Subject: [PATCH] osd: optionally periodically check pglog for corruption

...at least until we find this bug, periodically check pglog for corruption
and log it to the monitor when it happens.

Off by default.
---
 src/config.cc  |  1 +
 src/config.h   |  2 ++
 src/osd/OSD.cc |  4 +++
 src/osd/PG.cc  | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/osd/PG.h   |  1 +
 5 files changed, 76 insertions(+)

diff --git a/src/config.cc b/src/config.cc
index 5aa62f89d19bd..727f495e28dad 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -470,6 +470,7 @@ static struct config_option config_optionsp[] = {
 	OPTION(osd_recovery_max_active, 0, OPT_INT, 5),
 	OPTION(osd_auto_weight, 0, OPT_BOOL, false),
 	OPTION(osd_class_timeout, 0, OPT_FLOAT, 10.0),
+	OPTION(osd_check_for_log_corruption, 0, OPT_BOOL, false),
 	OPTION(filestore, 0, OPT_BOOL, false),
 	OPTION(filestore_max_sync_interval, 0, OPT_DOUBLE, 5),    // seconds
 	OPTION(filestore_min_sync_interval, 0, OPT_DOUBLE, .01),  // seconds
diff --git a/src/config.h b/src/config.h
index 68dca6f1b2c23..b21774e020780 100644
--- a/src/config.h
+++ b/src/config.h
@@ -339,6 +339,8 @@ struct md_config_t {
 
   bool osd_class_timeout;
 
+  bool osd_check_for_log_corruption;  // bleh
+
   // filestore
   bool filestore;
   double   filestore_max_sync_interval;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index add4926700b38..802ff5e4b48b9 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2533,6 +2533,10 @@ void OSD::activate_map(ObjectStore::Transaction& t, list<Context*>& tfin)
        it++) {
     PG *pg = it->second;
     pg->lock();
+
+    if (g_conf.osd_check_for_log_corruption)
+      pg->check_log_for_corruption(store);
+
     if (!osdmap->have_pg_pool(pg->info.pgid.pool())) {
       //pool is deleted!
       queue_pg_for_deletion(pg);
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 429bda0b600e6..0032c41afe160 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2276,6 +2276,74 @@ void PG::read_log(ObjectStore *store)
   dout(10) << "read_log done" << dendl;
 }
 
+bool PG::check_log_for_corruption(ObjectStore *store)
+{
+  OndiskLog bounds;
+  bufferlist blb;
+  store->collection_getattr(coll_t::build_pg_coll(info.pgid), "ondisklog", blb);
+  bufferlist::iterator p = blb.begin();
+  ::decode(bounds, p);
+
+  dout(10) << "check_log_for_corruption: tail " << bounds.tail << " head " << bounds.head
+	   << " block_map " << bounds.block_map << dendl;
+
+  stringstream ss;
+  ss << "CORRUPT pg " << info.pgid << " log: ";
+
+  bool ok = true;
+  uint64_t pos = 0;
+  if (bounds.head > 0) {
+    // read
+    struct stat st;
+    store->stat(meta_coll, log_oid, &st);
+    bufferlist bl;
+    store->read(meta_coll, log_oid, bounds.tail, bounds.length(), bl);
+    if (st.st_size != (int)bounds.head) {
+      ss << "mismatched bounds " << bounds.tail << ".." << bounds.head << " and file size " << st.st_size;
+      ok = false;
+    } else if (bl.length() < bounds.length()) {
+      dout(0) << " got " << bl.length() << " bytes, expected " 
+	      << bounds.tail << ".." << bounds.head << "="
+	      << bounds.length()
+	      << dendl;
+      ss << "short log, " << bl.length() << " bytes, expected " << bounds.length();
+      ok = false;
+    } else {
+      PG::Log::Entry e;
+      bufferlist::iterator p = bl.begin();
+      while (!p.end()) {
+	pos = bounds.tail + p.get_off();
+	try {
+	  ::decode(e, p);
+	}
+	catch (buffer::error *e) {
+	  dout(0) << "corrupt entry at " << pos << dendl;
+	  ss << "corrupt entry at offset " << pos;
+	  ok = false;
+	  break;
+	}
+	catch(std::bad_alloc a) {
+	  dout(0) << "corrupt entry at " << pos << dendl;
+	  ss << "corrupt entry at offset " << pos;
+	  ok = false;
+	  break;
+	}
+	dout(30) << " " << pos << " " << e << dendl;
+      }
+    }
+  }
+  if (!ok) {
+    stringstream f;
+    f << "/tmp/pglog_bad_" << info.pgid;
+    string filename;
+    getline(f, filename);
+    blb.write_file(filename.c_str(), 0644);
+    ss << ", saved to " << filename;
+    osd->logclient.log(LOG_ERROR, ss);
+  }
+  return ok;
+}
+
 
 
 void PG::read_state(ObjectStore *store)
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 73947e4593fc4..20c0f919c06c4 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -944,6 +944,7 @@ public:
   void append_log(ObjectStore::Transaction &t, bufferlist& bl,
 		  eversion_t log_version);
   void read_log(ObjectStore *store);
+  bool check_log_for_corruption(ObjectStore *store);
   void trim(ObjectStore::Transaction& t, eversion_t v);
   void trim_ondisklog_to(ObjectStore::Transaction& t, eversion_t v);
   void trim_peers();
-- 
2.39.5