From d713c7dad0adf8605c14cd8181c1b2c09b20fb8c Mon Sep 17 00:00:00 2001 From: David Zafman Date: Wed, 14 Mar 2018 16:07:01 -0700 Subject: [PATCH] osd rados command: Improve scrub handling of HashInfo (hinfo_key xattr) Fixes: http://tracker.ceph.com/issues/23364 Signed-off-by: David Zafman --- doc/rados/command/list-inconsistent-obj.json | 11 +- qa/standalone/scrub/osd-scrub-repair.sh | 427 ++++++++++++++++++- src/common/scrub_types.h | 9 + src/include/rados/rados_types.hpp | 18 +- src/osd/ECUtil.cc | 10 + src/osd/ECUtil.h | 2 + src/osd/PGBackend.cc | 33 +- src/tools/CMakeLists.txt | 3 +- src/tools/rados/rados.cc | 22 +- 9 files changed, 522 insertions(+), 13 deletions(-) diff --git a/doc/rados/command/list-inconsistent-obj.json b/doc/rados/command/list-inconsistent-obj.json index 859dc84ca2b5a..ec2ed0fd17026 100644 --- a/doc/rados/command/list-inconsistent-obj.json +++ b/doc/rados/command/list-inconsistent-obj.json @@ -69,7 +69,9 @@ "oi_attr_corrupted", "obj_size_oi_mismatch", "ss_attr_missing", - "ss_attr_corrupted" + "ss_attr_corrupted", + "hinfo_missing", + "hinfo_corrupted" ] }, "minItems": 0, @@ -86,7 +88,8 @@ "size_mismatch", "attr_value_mismatch", "attr_name_mismatch", - "snapset_inconsistency" + "snapset_inconsistency", + "hinfo_inconsistency" ] }, "minItems": 0, @@ -139,7 +142,9 @@ "oi_attr_corrupted", "obj_size_oi_mismatch", "ss_attr_missing", - "ss_attr_corrupted" + "ss_attr_corrupted", + "hinfo_missing", + "hinfo_corrupted" ] }, "minItems": 0, diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh index 7ba2283cd013a..e33445c984789 100755 --- a/qa/standalone/scrub/osd-scrub-repair.sh +++ b/qa/standalone/scrub/osd-scrub-repair.sh @@ -1698,7 +1698,7 @@ function corrupt_scrub_erasure() { local dir=$1 local allow_overwrites=$2 local poolname=ecpool - local total_objs=5 + local total_objs=7 setup $dir || return 1 run_mon $dir a || return 1 @@ -1760,6 +1760,23 @@ function corrupt_scrub_erasure() { objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 ;; + 6) + objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1 + echo -n bad-val > $dir/bad-val + objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1 + ;; + + 7) + local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS + echo $payload > $dir/DIFFERENT + rados --pool $poolname put $objname $dir/DIFFERENT || return 1 + + # Get hinfo_key from EOBJ1 + objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo + objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1 + rm -f $dir/hinfo + ;; + esac done @@ -2022,6 +2039,134 @@ function corrupt_scrub_erasure() { "nspace": "", "name": "EOBJ5" } + }, + { + "errors": [], + "object": { + "locator": "", + "name": "EOBJ6", + "nspace": "", + "snap": "head", + "version": 8 + }, + "selected_object_info": "3:4e671bad:::EOBJ6:head(65'7 client.4441.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "shards": [ + { + "attrs": [ + { + "Base64": true, + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" + }, + { + "Base64": true, + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + } + ], + "errors": [ + "hinfo_missing" + ], + "osd": 0, + "primary": false, + "shard": 2, + "size": 2048 + }, + { + "attrs": [ + { + "Base64": true, + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" + }, + { + "Base64": false, + "name": "hinfo_key", + "value": "bad-val" + }, + { + "Base64": true, + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + } + ], + "errors": [ + "hinfo_corrupted" + ], + "osd": 1, + "primary": true, + "shard": 0, + "size": 2048 + }, + { + "attrs": [ + { + "Base64": true, + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" + }, + { + "Base64": true, + "name": "hinfo_key", + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E" + }, + { + "Base64": true, + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + } + ], + "errors": [], + "osd": 2, + "primary": false, + "shard": 1, + "size": 2048 + } + ], + "union_shard_errors": [ + "hinfo_missing", + "hinfo_corrupted" + ] + }, + { + "errors": [ + "hinfo_inconsistency" + ], + "object": { + "locator": "", + "name": "EOBJ7", + "nspace": "", + "snap": "head", + "version": 10 + }, + "selected_object_info": "3:21a44c43:::EOBJ7:head(65'7 client.4441.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "shards": [ + { + "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f", + "errors": [], + "osd": 0, + "primary": false, + "shard": 2, + "size": 2048 + }, + { + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8", + "errors": [], + "osd": 1, + "primary": true, + "shard": 0, + "size": 2048 + }, + { + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8", + "errors": [], + "osd": 2, + "primary": false, + "shard": 1, + "size": 2048 + } + ], + "union_shard_errors": [] } ], "epoch": 0 @@ -2321,6 +2466,145 @@ EOF "nspace": "", "name": "EOBJ5" } + }, + { + "object": { + "name": "EOBJ6", + "nspace": "", + "locator": "", + "snap": "head", + "version": 8 + }, + "errors": [], + "union_shard_errors": [ + "read_error", + "hinfo_missing", + "hinfo_corrupted" + ], + "selected_object_info": "3:4e671bad:::EOBJ6:head(65'8 client.4432.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "read_error", + "hinfo_missing" + ], + "size": 2048, + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [ + "read_error", + "hinfo_corrupted" + ], + "size": 2048, + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "hinfo_key", + "value": "bad-val", + "Base64": false + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "hinfo_key", + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "Base64": true + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + } + ] + }, + { + "object": { + "name": "EOBJ7", + "nspace": "", + "locator": "", + "snap": "head", + "version": 10 + }, + "errors": [ + "hinfo_inconsistency" + ], + "union_shard_errors": [], + "selected_object_info": "3:21a44c43:::EOBJ7:head(75'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f" + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + } + ] } ], "epoch": 0 @@ -2639,6 +2923,147 @@ EOF "nspace": "", "name": "EOBJ5" } + }, + { + "object": { + "name": "EOBJ6", + "nspace": "", + "locator": "", + "snap": "head", + "version": 8 + }, + "errors": [], + "union_shard_errors": [ + "read_error", + "hinfo_missing", + "hinfo_corrupted" + ], + "selected_object_info": "3:4e671bad:::EOBJ6:head(63'8 client.4434.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "read_error", + "hinfo_missing" + ], + "size": 2048, + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [ + "read_error", + "hinfo_corrupted" + ], + "size": 2048, + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "hinfo_key", + "value": "bad-val", + "Base64": false + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x04cfa72f", + "attrs": [ + { + "name": "_", + "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + }, + { + "name": "hinfo_key", + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "Base64": true + }, + { + "name": "snapset", + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "Base64": true + } + ] + } + ] + }, + { + "object": { + "name": "EOBJ7", + "nspace": "", + "locator": "", + "snap": "head", + "version": 10 + }, + "errors": [ + "hinfo_inconsistency" + ], + "union_shard_errors": [ + "ec_hash_error" + ], + "selected_object_info": "3:21a44c43:::EOBJ7:head(73'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "ec_hash_error" + ], + "size": 2048, + "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f" + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x5b7455a8", + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x5b7455a8", + "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + } + ] } ], "epoch": 0 diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h index 2a17928344a37..4285c4637e56d 100644 --- a/src/common/scrub_types.h +++ b/src/common/scrub_types.h @@ -83,6 +83,12 @@ public: void set_obj_size_oi_mismatch() { errors |= err_t::OBJ_SIZE_OI_MISMATCH; } + void set_hinfo_missing() { + errors |= err_t::HINFO_MISSING; + } + void set_hinfo_corrupted() { + errors |= err_t::HINFO_CORRUPTED; + } void encode(bufferlist& bl) const; void decode(bufferlist::iterator& bp); }; @@ -120,6 +126,9 @@ struct inconsistent_obj_wrapper : librados::inconsistent_obj_t { void set_snapset_inconsistency() { errors |= obj_err_t::SNAPSET_INCONSISTENCY; } + void set_hinfo_inconsistency() { + errors |= obj_err_t::HINFO_INCONSISTENCY; + } void add_shard(const pg_shard_t& pgs, const shard_info_wrapper& shard); void set_auth_missing(const hobject_t& hoid, const map&, diff --git a/src/include/rados/rados_types.hpp b/src/include/rados/rados_types.hpp index 37e9d28545988..7d8d075e1ff93 100644 --- a/src/include/rados/rados_types.hpp +++ b/src/include/rados/rados_types.hpp @@ -64,11 +64,13 @@ struct err_t { OI_ATTR_CORRUPTED = 1 << 15, SS_ATTR_MISSING = 1 << 16, SS_ATTR_CORRUPTED = 1 << 17, - OBJ_SIZE_OI_MISMATCH = 1 << 18 + OBJ_SIZE_OI_MISMATCH = 1 << 18, + HINFO_MISSING = 1 << 19, + HINFO_CORRUPTED = 1 << 20 // When adding more here add to either SHALLOW_ERRORS or DEEP_ERRORS }; uint64_t errors = 0; - static constexpr uint64_t SHALLOW_ERRORS = SHARD_MISSING|SHARD_STAT_ERR|SIZE_MISMATCH_OI|OI_ATTR_MISSING|OI_ATTR_CORRUPTED|SS_ATTR_MISSING|SS_ATTR_CORRUPTED|OBJ_SIZE_OI_MISMATCH; + static constexpr uint64_t SHALLOW_ERRORS = SHARD_MISSING|SHARD_STAT_ERR|SIZE_MISMATCH_OI|OI_ATTR_MISSING|OI_ATTR_CORRUPTED|SS_ATTR_MISSING|SS_ATTR_CORRUPTED|OBJ_SIZE_OI_MISMATCH|HINFO_MISSING|HINFO_CORRUPTED; static constexpr uint64_t DEEP_ERRORS = SHARD_READ_ERR|DATA_DIGEST_MISMATCH_OI|OMAP_DIGEST_MISMATCH_OI|SHARD_EC_HASH_MISMATCH|SHARD_EC_SIZE_MISMATCH; bool has_shard_missing() const { return errors & SHARD_MISSING; @@ -115,6 +117,12 @@ struct err_t { bool has_obj_size_oi_mismatch() const { return errors & OBJ_SIZE_OI_MISMATCH; } + bool has_hinfo_missing() const { + return errors & HINFO_MISSING; + } + bool has_hinfo_corrupted() const { + return errors & HINFO_CORRUPTED; + } }; struct shard_info_t : err_t { @@ -152,10 +160,11 @@ struct obj_err_t { ATTR_VALUE_MISMATCH = 1 << 7, ATTR_NAME_MISMATCH = 1 << 8, SNAPSET_INCONSISTENCY = 1 << 9, + HINFO_INCONSISTENCY = 1 << 10, // When adding more here add to either SHALLOW_ERRORS or DEEP_ERRORS }; uint64_t errors = 0; - static constexpr uint64_t SHALLOW_ERRORS = OBJECT_INFO_INCONSISTENCY|SIZE_MISMATCH|ATTR_VALUE_MISMATCH|ATTR_NAME_MISMATCH|SNAPSET_INCONSISTENCY; + static constexpr uint64_t SHALLOW_ERRORS = OBJECT_INFO_INCONSISTENCY|SIZE_MISMATCH|ATTR_VALUE_MISMATCH|ATTR_NAME_MISMATCH|SNAPSET_INCONSISTENCY|HINFO_INCONSISTENCY; static constexpr uint64_t DEEP_ERRORS = DATA_DIGEST_MISMATCH|OMAP_DIGEST_MISMATCH; bool has_object_info_inconsistency() const { return errors & OBJECT_INFO_INCONSISTENCY; @@ -184,6 +193,9 @@ struct obj_err_t { bool has_snapset_inconsistency() const { return errors & SNAPSET_INCONSISTENCY; } + bool has_hinfo_inconsistency() const { + return errors & HINFO_INCONSISTENCY; + } }; struct inconsistent_obj_t : obj_err_t { diff --git a/src/osd/ECUtil.cc b/src/osd/ECUtil.cc index 6d538f0003404..a6704953fc75f 100644 --- a/src/osd/ECUtil.cc +++ b/src/osd/ECUtil.cc @@ -206,6 +206,16 @@ void ECUtil::HashInfo::dump(Formatter *f) const f->close_section(); } +namespace ECUtil { +std::ostream& operator<<(std::ostream& out, const HashInfo& hi) +{ + ostringstream hashes; + for (auto hash: hi.cumulative_shard_hashes) + hashes << " " << hex << hash; + return out << "tcs=" << hi.total_chunk_size << hashes.str(); +} +} + void ECUtil::HashInfo::generate_test_instances(list& o) { o.push_back(new HashInfo(3)); diff --git a/src/osd/ECUtil.h b/src/osd/ECUtil.h index 94ce1a1a4d195..fb2e8ad283c39 100644 --- a/src/osd/ECUtil.h +++ b/src/osd/ECUtil.h @@ -15,6 +15,7 @@ #ifndef ECUTIL_H #define ECUTIL_H +#include #include "erasure-code/ErasureCodeInterface.h" #include "include/buffer_fwd.h" #include "include/assert.h" @@ -155,6 +156,7 @@ public: *this = rhs; projected_total_chunk_size = ptcs; } + friend std::ostream& operator<<(std::ostream& out, const HashInfo& hi); }; typedef ceph::shared_ptr HashInfoRef; diff --git a/src/osd/PGBackend.cc b/src/osd/PGBackend.cc index bd4d98681c893..dde1f3965d85d 100644 --- a/src/osd/PGBackend.cc +++ b/src/osd/PGBackend.cc @@ -707,7 +707,7 @@ bool PGBackend::be_compare_scrub_objects( i != auth.attrs.end(); ++i) { // We check system keys seperately - if (i->first == OI_ATTR || i->first == SS_ATTR) + if (i->first == OI_ATTR || i->first[0] != '_') continue; if (!candidate.attrs.count(i->first)) { if (error != CLEAN) @@ -727,7 +727,7 @@ bool PGBackend::be_compare_scrub_objects( i != candidate.attrs.end(); ++i) { // We check system keys seperately - if (i->first == OI_ATTR || i->first == SS_ATTR) + if (i->first == OI_ATTR || i->first[0] != '_') continue; if (!auth.attrs.count(i->first)) { if (error != CLEAN) @@ -759,7 +759,7 @@ map::const_iterator inconsistent_obj_wrapper &object_error) { eversion_t auth_version; - bufferlist first_oi_bl, first_ss_bl; + bufferlist first_oi_bl, first_ss_bl, first_hk_bl; // Create list of shards with primary first so it will be auth copy all // other things being equal. @@ -802,7 +802,7 @@ map::const_iterator bufferlist bl; map::iterator k; SnapSet ss; - bufferlist ss_bl; + bufferlist ss_bl, hk_bl; if (i->second.stat_error) { shard_info.set_stat_error(); @@ -838,6 +838,31 @@ map::const_iterator } } + if (parent->get_pool().is_erasure()) { + ECUtil::HashInfo hi; + k = i->second.attrs.find(ECUtil::get_hinfo_key()); + if (k == i->second.attrs.end()) { + shard_info.set_hinfo_missing(); + error_string += " hinfo_key_missing"; + } else { + hk_bl.push_back(k->second); + try { + bufferlist::iterator bliter = hk_bl.begin(); + decode(hi, bliter); + if (first_hk_bl.length() == 0) { + first_hk_bl.append(hk_bl); + } else if (!object_error.has_hinfo_inconsistency() && !hk_bl.contents_equal(first_hk_bl)) { + object_error.set_hinfo_inconsistency(); + error_string += " hinfo_inconsistency"; + } + } catch (...) { + // invalid snapset, probably corrupt + shard_info.set_hinfo_corrupted(); + error_string += " hinfo_corrupted"; + } + } + } + k = i->second.attrs.find(OI_ATTR); if (k == i->second.attrs.end()) { // no object info on object, probably corrupt diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 6e30ed2e6d973..e0844ec0c7a1f 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -4,7 +4,8 @@ set(rados_srcs rados/RadosImport.cc rados/PoolDump.cc ${PROJECT_SOURCE_DIR}/src/common/util.cc - ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc) + ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc + ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc) add_executable(rados ${rados_srcs}) target_link_libraries(rados librados global ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS} radosstriper) install(TARGETS rados DESTINATION bin) diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index 4fd2d696508eb..a9ebc4efea8b4 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -52,6 +52,8 @@ using namespace libradosstriper; #include "PoolDump.h" #include "RadosImport.h" +#include "osd/ECUtil.h" + using namespace librados; using ceph::util::generate_random_number; @@ -1348,6 +1350,10 @@ static void dump_errors(const err_t &err, Formatter &f, const char *name) f.dump_string("error", "ss_attr_missing"); if (err.has_ss_attr_corrupted()) f.dump_string("error", "ss_attr_corrupted"); + if (err.has_hinfo_missing()) + f.dump_string("error", "hinfo_missing"); + if (err.has_hinfo_corrupted()) + f.dump_string("error", "hinfo_corrupted"); f.close_section(); } @@ -1389,11 +1395,23 @@ static void dump_shard(const shard_info_t& shard, decode(ss, bliter); // Can't be corrupted f.dump_stream("snapset") << ss; } + if (!shard.has_hinfo_missing() && !shard.has_hinfo_corrupted() && + inc.has_hinfo_inconsistency()) { + ECUtil::HashInfo hi; + bufferlist bl; + map::iterator k = (const_cast(shard)).attrs.find(ECUtil::get_hinfo_key()); + assert(k != shard.attrs.end()); // Can't be missing + bufferlist::iterator bliter = k->second.begin(); + decode(hi, bliter); // Can't be corrupted + f.dump_stream("hashinfo") << hi; + } if (inc.has_attr_name_mismatch() || inc.has_attr_value_mismatch() || inc.union_shards.has_oi_attr_missing() || inc.union_shards.has_oi_attr_corrupted() || inc.union_shards.has_ss_attr_missing() - || inc.union_shards.has_ss_attr_corrupted()) { + || inc.union_shards.has_ss_attr_corrupted() + || inc.union_shards.has_hinfo_missing() + || inc.union_shards.has_hinfo_corrupted()) { f.open_array_section("attrs"); for (auto kv : shard.attrs) { f.open_object_section("attr"); @@ -1424,6 +1442,8 @@ static void dump_obj_errors(const obj_err_t &err, Formatter &f) f.dump_string("error", "attr_name_mismatch"); if (err.has_snapset_inconsistency()) f.dump_string("error", "snapset_inconsistency"); + if (err.has_hinfo_inconsistency()) + f.dump_string("error", "hinfo_inconsistency"); f.close_section(); } -- 2.39.5