common: add osd_distrust_data_digest option

If we have reason to distrust the stored full-object oi data digests,
this option will ignore them (and opportunistically clear them).  It
basically affects the same behaviors that osd_skip_data_digest except
the previous option only takes effect if bluestore is in use, while this
option is unconditional.

This serves a workaround for clusters that suffered the 12.2.6 bug that
produced bad full-object digests and *also* had PGs' primaries move from
bluestore to filestore nodes.  In those situations, this option can be
set to true.

After all OSDs are upgrade and all PGs come back with a clean deep
scrub this option can be disabled again.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index e6e609f8aad466a6d90f011b366aea7a19f8b078..e2ac86444322c6c996dc7b3cb01db69257aca3c1 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -790,6 +790,7 @@ OPTION(osd_deep_scrub_stride, OPT_INT)
 OPTION(osd_deep_scrub_keys, OPT_INT)
 OPTION(osd_deep_scrub_update_digest_min_age, OPT_INT)   // objects must be this old (seconds) before we update the whole-object digest on scrub
 OPTION(osd_skip_data_digest, OPT_BOOL)
+OPTION(osd_distrust_data_digest, OPT_BOOL)
 OPTION(osd_deep_scrub_large_omap_object_key_threshold, OPT_U64)
 OPTION(osd_deep_scrub_large_omap_object_value_sum_threshold, OPT_U64)
 OPTION(osd_class_dir, OPT_STR) // where rados plugins are stored

diff --git a/src/common/options.cc b/src/common/options.cc
index 7100457059053531796120a3bb31ebdd367b12bb..7e426c29c3fb368bfefe4e7d27d731e05137f8ec 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1989,6 +1989,10 @@ std::vector<Option> get_global_options() {
     .set_default(true)
     .set_description(""),
 
+    Option("osd_distrust_data_digest", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_description("Do not trust stored data_digest (due to previous bug or corruption)"),
+
     Option("osd_op_queue", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("wpq")
     .set_enum_allowed( { "wpq", "prioritized", "mclock_opclass", "mclock_client", "debug_random" } )

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index a4e4686e545097e676c91d03ced8c26bc3e1f26d..aaf9136a45e672b91acdbfcc2dc6f7c3ed3e3f28 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -4532,8 +4532,9 @@ int PrimaryLogPG::do_checksum(OpContext *ctx, OSDOp& osd_op,
                              bufferlist::iterator *bl_it)
 {
   dout(20) << __func__ << dendl;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   auto& op = osd_op.op;
   if (op.checksum.chunk_size > 0) {
@@ -4737,8 +4738,9 @@ int PrimaryLogPG::do_extent_cmp(OpContext *ctx, OSDOp& osd_op)
 {
   dout(20) << __func__ << dendl;
   ceph_osd_op& op = osd_op.op;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   auto& oi = ctx->new_obs.oi;
   uint64_t size = oi.size;
@@ -4822,8 +4824,9 @@ int PrimaryLogPG::do_read(OpContext *ctx, OSDOp& osd_op) {
   __u32 seq = oi.truncate_seq;
   uint64_t size = oi.size;
   bool trimmed_read = false;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   // are we beyond truncate_size?
   if ( (seq < op.extent.truncate_seq) &&
@@ -4909,8 +4912,9 @@ int PrimaryLogPG::do_sparse_read(OpContext *ctx, OSDOp& osd_op) {
   auto& op = osd_op.op;
   auto& oi = ctx->new_obs.oi;
   auto& soid = oi.soid;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   if (op.extent.truncate_seq) {
     dout(0) << "sparse_read does not support truncation sequence " << dendl;
@@ -5058,8 +5062,9 @@ int PrimaryLogPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
   ObjectState& obs = ctx->new_obs;
   object_info_t& oi = obs.oi;
   const hobject_t& soid = oi.soid;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   PGTransaction* t = ctx->op_t.get();
 
@@ -7849,8 +7854,9 @@ int PrimaryLogPG::do_copy_get(OpContext *ctx, bufferlist::iterator& bp,
   int result = 0;
   object_copy_cursor_t cursor;
   uint64_t out_max;
-  bool skip_data_digest = osd->store->has_builtin_csum() &&
-    g_conf->osd_skip_data_digest;
+  bool skip_data_digest =
+    (osd->store->has_builtin_csum() && g_conf->osd_skip_data_digest) ||
+    g_conf->osd_distrust_data_digest;
 
   try {
     ::decode(cursor, bp);