osd: optional verify that sparse_read holes are zero-filled

This should help us track down/verify #2535.  It seems to happen on several
different systems, but we haven't figured out which ones yet.

This detects the bug, but does not attempt to correct it.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 93864d8196e4b24f71dc75d2d1e49ca0093e2bc6..bd6c39aedc62f7fce538db104e3d166008152874 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -335,6 +335,7 @@ OPTION(osd_min_pg_log_entries, OPT_U32, 1000) // number of entries to keep in th
 OPTION(osd_op_complaint_time, OPT_FLOAT, 30) // how many seconds old makes an op complaint-worthy
 OPTION(osd_command_max_records, OPT_INT, 256)
 OPTION(osd_op_log_threshold, OPT_INT, 5) // how many op log messages to show in one go
+OPTION(osd_verify_sparse_read_holes, OPT_BOOL, false)  // read fiemap-reported holes and verify they are zeros
 OPTION(filestore, OPT_BOOL, false)
 OPTION(filestore_debug_omap_check, OPT_BOOL, 0) // Expensive debugging check on sync
 // Use omap for xattrs for attrs over

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 0463cd1d64bfa85ff761b7b25a48b031b1aa001e..2473be6264584a7d4cc80a31fb5ab5920d09bfa3 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1709,7 +1709,20 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
         ::decode(m, iter);
         map<uint64_t, uint64_t>::iterator miter;
         bufferlist data_bl;
+       uint64_t last = op.extent.offset;
         for (miter = m.begin(); miter != m.end(); ++miter) {
+         // verify hole?
+         if (g_conf->osd_verify_sparse_read_holes &&
+             last < miter->first) {
+           bufferlist t;
+           uint64_t len = miter->first - last;
+           r = osd->store->read(coll, soid, last, len, t);
+           if (!t.is_zero()) {
+             osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
+                               << last << "~" << len << "\n";
+           }
+         }
+
           bufferlist tmpbl;
           r = osd->store->read(coll, soid, miter->first, miter->second, tmpbl);
           if (r < 0)
@@ -1720,8 +1733,23 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
           total_read += r;
           dout(10) << "sparse-read " << miter->first << "@" << miter->second << dendl;
          data_bl.claim_append(tmpbl);
+         last = miter->first + r;
         }
 
+       // verify trailing hole?
+       if (g_conf->osd_verify_sparse_read_holes) {
+         uint64_t end = MIN(op.extent.offset + op.extent.length, oi.size);
+         if (last < end) {
+           bufferlist t;
+           uint64_t len = end - last;
+           r = osd->store->read(coll, soid, last, len, t);
+           if (!t.is_zero()) {
+             osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
+                               << last << "~" << len << "\n";
+           }
+         }
+       }
+
         if (r < 0) {
           result = r;
           break;