From: David Zafman Date: Fri, 16 Mar 2018 06:59:52 +0000 (-0700) Subject: osd rados command: list-inconsistent-obj attribute improvements X-Git-Tag: v12.2.5~11^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d4473fc5da3d4d77734ce3c9a6d120b191e28cf;p=ceph.git osd rados command: list-inconsistent-obj attribute improvements System attributes shown as "object_info", "snapset" and "hashinfo" Only output user attributes as "attrs" Drop leading undescore "_" for user attribute keys Improve logic as to when to show user attributes or specific system attributes Signed-off-by: David Zafman (cherry picked from commit 982509514c0c990dcdf7f35725dece61714bfa59) Conflicts: qa/standalone/scrub/osd-scrub-repair.sh (trivial) src/osd/osd_types.cc (trivial) src/tools/rados/rados.cc (uses ::decode()) --- diff --git a/doc/rados/command/list-inconsistent-obj.json b/doc/rados/command/list-inconsistent-obj.json index 5de262877f41..637e3ed8f74f 100644 --- a/doc/rados/command/list-inconsistent-obj.json +++ b/doc/rados/command/list-inconsistent-obj.json @@ -50,7 +50,9 @@ ] }, "selected_object_info": { - "type": "string" + "type": "object", + "description": "Selected object information", + "additionalProperties": true }, "union_shard_errors": { "description": "Union of all shard errors", @@ -103,7 +105,40 @@ "type": "object", "properties": { "object_info": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Object information", + "additionalProperties": true + } + ] + }, + "snapset": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Snap set information", + "additionalProperties": true + } + ] + }, + "hashinfo": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Erasure code hash information", + "additionalProperties": true + } + ] }, "shard": { "type": "integer" @@ -172,11 +207,11 @@ "value", "Base64" ], - "additionalProperties": false, - "minItems": 1 + "additionalProperties": false } } }, + "additionalProperties": false, "required": [ "osd", "primary", diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh index 86e7f42fbb7c..f295284e0183 100755 --- a/qa/standalone/scrub/osd-scrub-repair.sh +++ b/qa/standalone/scrub/osd-scrub-repair.sh @@ -29,16 +29,19 @@ fi # Set to "yes" in order to ignore diff errors and save results to update test getjson="no" -# Ignore the epoch and filter out the attr '_' value because it has date information and won't match -if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here - jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"' -else - jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"' -fi -sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' +# Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info. +jqfilter='(. | (.inconsistents[].selected_object_info | .mtime ) |= "----Stripped-----") + | (. | (.inconsistents[].selected_object_info | .local_mtime ) |= "----Stripped-----" ) + | (. | (.inconsistents[].selected_object_info | .last_reqid ) |= "----Stripped-----") + | (. | (.inconsistents[].selected_object_info | .version ) |= "----Stripped-----" ) + | (. | (.inconsistents[].selected_object_info | .prior_version ) |= "----Stripped-----" ) + | ( . | (.inconsistents[].shards[].object_info | .mtime ) |= "----Stripped-----" ) + | ( . | (.inconsistents[].shards[].object_info | .local_mtime ) |= "----Stripped-----" ) + | ( . | (.inconsistents[].shards[].object_info | .last_reqid ) |= "----Stripped-----" ) + | ( .| (.inconsistents[].shards[].object_info | .version ) |= "----Stripped-----" ) + | ( .| (.inconsistents[].shards[].object_info | .prior_version ) |= "----Stripped-----" )' -# Remove items are not consistent across runs, the pg interval and client -sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' +sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' function run() { local dir=$1 @@ -626,7 +629,7 @@ function TEST_corrupt_scrub_replicated() { # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -638,6 +641,41 @@ function TEST_corrupt_scrub_replicated() { "primary": false }, { + "object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "errors": [ "size_mismatch_info", @@ -647,7 +685,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "size_mismatch_info", "obj_size_info_mismatch" @@ -679,7 +751,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ12", + "key": "", + "snapid": -2, + "hash": 3920199997, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'56", + "prior_version": "43'36", + "last_reqid": "osd.1.0:55", + "user_version": 36, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -708,7 +814,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ13", + "key": "", + "snapid": -2, + "hash": 2682806379, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'59", + "prior_version": "45'39", + "last_reqid": "osd.1.0:58", + "user_version": 39, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -724,19 +864,8 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { + "object_info": "bad-val", "size": 7, - "attrs": [ - { - "Base64": false, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "errors": [ "info_corrupted" ], @@ -745,13 +874,6 @@ function TEST_corrupt_scrub_replicated() { }, { "size": 7, - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "errors": [ "info_missing" ], @@ -775,31 +897,47 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" + "object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 7, "errors": [], "osd": 0, "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "size": 7, "errors": [ "info_missing" @@ -808,7 +946,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "info_missing" ], @@ -832,13 +1004,6 @@ function TEST_corrupt_scrub_replicated() { }, "shards": [ { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - } - ], "errors": [ "snapset_missing" ], @@ -847,23 +1012,12 @@ function TEST_corrupt_scrub_replicated() { "size": 7 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - }, - { - "Base64": false, - "name": "snapset", - "value": "bad-val" - } - ], "errors": [ "snapset_corrupted" ], "osd": 1, "primary": true, + "snapset": "bad-val", "size": 7 } ], @@ -888,7 +1042,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ3", + "key": "", + "snapid": -2, + "hash": 625845583, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'61", + "prior_version": "25'9", + "last_reqid": "osd.1.0:60", + "user_version": 9, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -905,25 +1093,15 @@ function TEST_corrupt_scrub_replicated() { "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-ROBJ8" } ], "size": 7, @@ -933,25 +1111,15 @@ function TEST_corrupt_scrub_replicated() { }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-ROBJ8" } ], "size": 7, @@ -960,7 +1128,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ8", + "key": "", + "snapid": -2, + "hash": 2359695969, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "79'66", + "prior_version": "79'65", + "last_reqid": "client.4554.0:1", + "user_version": 66, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -977,14 +1179,82 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { - "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "95'67", + "prior_version": "51'64", + "last_reqid": "client.4649.0:1", + "user_version": 67, + "size": 1, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 727918093, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 1, "errors": [], "osd": 0, "primary": false }, { - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'64", + "prior_version": "37'27", + "last_reqid": "osd.1.0:63", + "user_version": 27, + "size": 7, + "mtime": "2018-04-05 14:33:25.352485", + "local_mtime": "2018-04-05 14:33:25.353746", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 1, "errors": [ "obj_size_info_mismatch" @@ -993,7 +1263,41 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "95'67", + "prior_version": "51'64", + "last_reqid": "client.4649.0:1", + "user_version": 67, + "size": 1, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 727918093, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "obj_size_info_mismatch" ], @@ -1013,7 +1317,7 @@ function TEST_corrupt_scrub_replicated() { } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then @@ -1056,7 +1360,7 @@ EOF # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -1070,6 +1374,41 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x2d4a11c2", "omap_digest": "0xf5fba2c6", "size": 9, @@ -1082,7 +1421,41 @@ EOF "primary": true } ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "data_digest_mismatch_info", "size_mismatch_info", @@ -1155,7 +1528,41 @@ EOF "primary": true } ], - "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ11", + "key": "", + "snapid": -2, + "hash": 1828574689, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'52", + "prior_version": "41'33", + "last_reqid": "osd.1.0:51", + "user_version": 33, + "size": 7, + "mtime": "2018-04-05 14:33:26.761286", + "local_mtime": "2018-04-05 14:33:26.762368", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error" ], @@ -1186,7 +1593,41 @@ EOF "primary": true } ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ12", + "key": "", + "snapid": -2, + "hash": 3920199997, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'56", + "prior_version": "43'36", + "last_reqid": "osd.1.0:55", + "user_version": 36, + "size": 7, + "mtime": "2018-04-05 14:33:27.460958", + "local_mtime": "2018-04-05 14:33:27.462109", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -1233,18 +1674,7 @@ EOF { "shards": [ { - "attrs": [ - { - "Base64": false, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "object_info": "bad-val", "data_digest": "0x2ddbf8f5", "omap_digest": "0x4f14f849", "size": 7, @@ -1255,13 +1685,6 @@ EOF "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "data_digest": "0x2ddbf8f5", "omap_digest": "0x4f14f849", "size": 7, @@ -1288,18 +1711,41 @@ EOF { "shards": [ { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" + "object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x2ddbf8f5", "omap_digest": "0x2d2a4d6e", "size": 7, @@ -1308,13 +1754,6 @@ EOF "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "data_digest": "0x2ddbf8f5", "omap_digest": "0x2d2a4d6e", "size": 7, @@ -1325,7 +1764,41 @@ EOF "primary": true } ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "info_missing" ], @@ -1349,13 +1822,6 @@ EOF }, "shards": [ { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - } - ], "data_digest": "0x2ddbf8f5", "errors": [ "snapset_missing" @@ -1366,18 +1832,7 @@ EOF "size": 7 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - }, - { - "Base64": false, - "name": "snapset", - "value": "bad-val" - } - ], + "snapset": "bad-val", "data_digest": "0x2ddbf8f5", "errors": [ "snapset_corrupted" @@ -1414,7 +1869,41 @@ EOF "primary": true } ], - "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ2", + "key": "", + "snapid": -2, + "hash": 2026323607, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'60", + "prior_version": "23'6", + "last_reqid": "osd.1.0:59", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:33:20.498756", + "local_mtime": "2018-04-05 14:33:20.499704", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "data_digest_mismatch_info" ], @@ -1447,7 +1936,41 @@ EOF "primary": true } ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ3", + "key": "", + "snapid": -2, + "hash": 625845583, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'61", + "prior_version": "25'9", + "last_reqid": "osd.1.0:60", + "user_version": 9, + "size": 7, + "mtime": "2018-04-05 14:33:21.189382", + "local_mtime": "2018-04-05 14:33:21.190446", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -1481,7 +2004,41 @@ EOF "primary": true } ], - "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ4", + "key": "", + "snapid": -2, + "hash": 2360875311, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'62", + "prior_version": "27'12", + "last_reqid": "osd.1.0:61", + "user_version": 12, + "size": 7, + "mtime": "2018-04-05 14:33:21.862313", + "local_mtime": "2018-04-05 14:33:21.863261", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "omap_digest_mismatch_oi" ], @@ -1517,7 +2074,41 @@ EOF "primary": true } ], - "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ5", + "key": "", + "snapid": -2, + "hash": 2334915887, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'63", + "prior_version": "29'15", + "last_reqid": "osd.1.0:62", + "user_version": 15, + "size": 7, + "mtime": "2018-04-05 14:33:22.589300", + "local_mtime": "2018-04-05 14:33:22.590376", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "omap_digest_mismatch_oi" ], @@ -1553,7 +2144,41 @@ EOF "primary": true } ], - "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ6", + "key": "", + "snapid": -2, + "hash": 390610085, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'54", + "prior_version": "31'18", + "last_reqid": "osd.1.0:53", + "user_version": 18, + "size": 7, + "mtime": "2018-04-05 14:33:23.289188", + "local_mtime": "2018-04-05 14:33:23.290130", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "omap_digest_mismatch_oi" ], @@ -1589,7 +2214,41 @@ EOF "primary": true } ], - "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ7", + "key": "", + "snapid": -2, + "hash": 3529485009, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'53", + "prior_version": "33'21", + "last_reqid": "osd.1.0:52", + "user_version": 21, + "size": 7, + "mtime": "2018-04-05 14:33:23.979658", + "local_mtime": "2018-04-05 14:33:23.980731", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "omap_digest_mismatch_oi" ], @@ -1608,25 +2267,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-ROBJ8" } ], "data_digest": "0x2ddbf8f5", @@ -1638,25 +2287,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-ROBJ8" } ], "data_digest": "0x2ddbf8f5", @@ -1667,7 +2306,41 @@ EOF "primary": true } ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ8", + "key": "", + "snapid": -2, + "hash": 2359695969, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "79'66", + "prior_version": "79'65", + "last_reqid": "client.4554.0:1", + "user_version": 66, + "size": 7, + "mtime": "2018-04-05 14:34:05.598688", + "local_mtime": "2018-04-05 14:34:05.599698", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -1684,7 +2357,41 @@ EOF { "shards": [ { - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'64", + "prior_version": "37'27", + "last_reqid": "osd.1.0:63", + "user_version": 27, + "size": 7, + "mtime": "2018-04-05 14:33:25.352485", + "local_mtime": "2018-04-05 14:33:25.353746", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x1f26fb26", "omap_digest": "0x2eecc539", "size": 3, @@ -1695,7 +2402,41 @@ EOF "primary": false }, { - "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "119'68", + "prior_version": "51'64", + "last_reqid": "client.4834.0:1", + "user_version": 68, + "size": 3, + "mtime": "2018-04-05 14:35:01.500659", + "local_mtime": "2018-04-05 14:35:01.502117", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 522648358, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x1f26fb26", "omap_digest": "0x2eecc539", "size": 3, @@ -1704,7 +2445,41 @@ EOF "primary": true } ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "119'68", + "prior_version": "51'64", + "last_reqid": "client.4834.0:1", + "user_version": 68, + "size": 3, + "mtime": "2018-04-05 14:35:01.500659", + "local_mtime": "2018-04-05 14:35:01.502117", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 522648358, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "obj_size_info_mismatch" ], @@ -1724,7 +2499,7 @@ EOF } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then @@ -1844,7 +2619,7 @@ function corrupt_scrub_erasure() { # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -1857,6 +2632,40 @@ function corrupt_scrub_erasure() { "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ @@ -1874,7 +2683,40 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "size_mismatch_info", "obj_size_info_mismatch" @@ -1915,7 +2757,40 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4252.0:1", + "user_version": 3, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -1932,30 +2807,15 @@ function corrupt_scrub_erasure() { "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "size": 2048, @@ -1971,30 +2831,15 @@ function corrupt_scrub_erasure() { "errors": [], "size": 2048, "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ] }, @@ -2005,35 +2850,53 @@ function corrupt_scrub_erasure() { "errors": [], "size": 2048, "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ] } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4294.0:1", + "user_version": 6, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2057,6 +2920,40 @@ function corrupt_scrub_erasure() { "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "shard": 0, "errors": [ @@ -2074,7 +2971,40 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "size_mismatch_info", "obj_size_info_mismatch" @@ -2099,21 +3029,42 @@ function corrupt_scrub_erasure() { "snap": "head", "version": 8 }, - "selected_object_info": "3:4e671bad:::EOBJ6:head(65'7 client.4441.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4418.0:1", + "user_version": 8, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" - }, - { - "Base64": true, - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" - } - ], "errors": [ "hinfo_missing" ], @@ -2123,54 +3074,38 @@ function corrupt_scrub_erasure() { "size": 2048 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" - }, - { - "Base64": false, - "name": "hinfo_key", - "value": "bad-val" - }, - { - "Base64": true, - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" - } - ], "errors": [ "hinfo_corrupted" ], "osd": 1, "primary": true, "shard": 0, + "hashinfo": "bad-val", "size": 2048 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=" - }, - { - "Base64": true, - "name": "hinfo_key", - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E" - }, - { - "Base64": true, - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=" - } - ], "errors": [], "osd": 2, "primary": false, "shard": 1, - "size": 2048 + "size": 2048, + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } } ], "union_shard_errors": [ @@ -2189,10 +3124,59 @@ function corrupt_scrub_erasure() { "snap": "head", "version": 10 }, - "selected_object_info": "3:21a44c43:::EOBJ7:head(65'7 client.4441.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4482.0:1", + "user_version": 10, + "size": 34, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 325996071, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { - "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, "errors": [], "osd": 0, "primary": false, @@ -2200,7 +3184,23 @@ function corrupt_scrub_erasure() { "size": 2048 }, { - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, "errors": [], "osd": 1, "primary": true, @@ -2208,7 +3208,23 @@ function corrupt_scrub_erasure() { "size": 2048 }, { - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, "errors": [], "osd": 2, "primary": false, @@ -2223,7 +3239,7 @@ function corrupt_scrub_erasure() { } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then @@ -2249,7 +3265,7 @@ EOF if [ "$allow_overwrites" = "true" ] then - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -2264,6 +3280,40 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:31:33.837147", + "local_mtime": "2018-04-05 14:31:33.840763", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ @@ -2284,7 +3334,40 @@ EOF "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:31:33.837147", + "local_mtime": "2018-04-05 14:31:33.840763", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error", "size_mismatch_info", @@ -2330,7 +3413,40 @@ EOF "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4252.0:1", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:31:46.841145", + "local_mtime": "2018-04-05 14:31:46.844996", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -2347,30 +3463,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x00000000", @@ -2383,30 +3484,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x00000000", @@ -2419,30 +3505,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ], "data_digest": "0x00000000", @@ -2454,7 +3525,40 @@ EOF "primary": false } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4294.0:1", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:31:54.663622", + "local_mtime": "2018-04-05 14:31:54.664527", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2482,6 +3586,40 @@ EOF { "data_digest": "0x00000000", "omap_digest": "0xffffffff", + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:32:12.929161", + "local_mtime": "2018-04-05 14:32:12.934707", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "errors": [ "size_mismatch_info", @@ -2501,7 +3639,40 @@ EOF "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:32:12.929161", + "local_mtime": "2018-04-05 14:32:12.934707", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "size_mismatch_info", "obj_size_info_mismatch" @@ -2531,7 +3702,40 @@ EOF "hinfo_missing", "hinfo_corrupted" ], - "selected_object_info": "3:4e671bad:::EOBJ6:head(65'8 client.4432.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4418.0:1", + "user_version": 8, + "size": 7, + "mtime": "2018-04-05 14:32:20.634116", + "local_mtime": "2018-04-05 14:32:20.637999", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, @@ -2541,19 +3745,7 @@ EOF "read_error", "hinfo_missing" ], - "size": 2048, - "attrs": [ - { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - }, - { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - } - ] + "size": 2048 }, { "osd": 1, @@ -2564,23 +3756,7 @@ EOF "hinfo_corrupted" ], "size": 2048, - "attrs": [ - { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - }, - { - "name": "hinfo_key", - "value": "bad-val", - "Base64": false - }, - { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - } - ] + "hashinfo": "bad-val" }, { "osd": 2, @@ -2590,23 +3766,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x00000000", - "attrs": [ + "hashinfo": { + "cumulative_shard_hashes": [ { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true + "hash": 80717615, + "shard": 0 }, { - "name": "hinfo_key", - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "Base64": true + "hash": 1534491824, + "shard": 1 }, { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true + "hash": 80717615, + "shard": 2 } - ] + ], + "total_chunk_size": 2048 + } } ] }, @@ -2622,7 +3798,40 @@ EOF "hinfo_inconsistency" ], "union_shard_errors": [], - "selected_object_info": "3:21a44c43:::EOBJ7:head(75'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4482.0:1", + "user_version": 10, + "size": 34, + "mtime": "2018-04-05 14:32:33.058782", + "local_mtime": "2018-04-05 14:32:33.059679", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 325996071, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, @@ -2632,7 +3841,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x00000000", - "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } }, { "osd": 1, @@ -2642,7 +3867,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x00000000", - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } }, { "osd": 2, @@ -2652,7 +3893,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x00000000", - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } } ] } @@ -2663,7 +3920,7 @@ EOF else - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -2678,6 +3935,40 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4192.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:30:10.688009", + "local_mtime": "2018-04-05 14:30:10.691774", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ @@ -2698,7 +3989,40 @@ EOF "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4192.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:30:10.688009", + "local_mtime": "2018-04-05 14:30:10.691774", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error", "size_mismatch_info", @@ -2745,7 +4069,40 @@ EOF "primary": false } ], - "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ2", + "key": "", + "snapid": -2, + "hash": 562812377, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "33'2", + "prior_version": "0'0", + "last_reqid": "client.4224.0:1", + "user_version": 2, + "size": 7, + "mtime": "2018-04-05 14:30:14.152945", + "local_mtime": "2018-04-05 14:30:14.154014", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "ec_hash_error" ], @@ -2787,7 +4144,40 @@ EOF "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4258.0:1", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:30:18.875544", + "local_mtime": "2018-04-05 14:30:18.880153", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -2804,30 +4194,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x04cfa72f", @@ -2847,30 +4222,15 @@ EOF "omap_digest": "0xffffffff", "data_digest": "0x04cfa72f", "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ] }, @@ -2883,35 +4243,53 @@ EOF "omap_digest": "0xffffffff", "data_digest": "0x04cfa72f", "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ] } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4296.0:1", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:30:22.271983", + "local_mtime": "2018-04-05 14:30:22.272840", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2937,6 +4315,40 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4384.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:30:35.162395", + "local_mtime": "2018-04-05 14:30:35.166390", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "shard": 0, "errors": [ @@ -2957,7 +4369,40 @@ EOF "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4384.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:30:35.162395", + "local_mtime": "2018-04-05 14:30:35.166390", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "size_mismatch_info", "ec_size_error", @@ -2988,7 +4433,40 @@ EOF "hinfo_missing", "hinfo_corrupted" ], - "selected_object_info": "3:4e671bad:::EOBJ6:head(63'8 client.4434.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4420.0:1", + "user_version": 8, + "size": 7, + "mtime": "2018-04-05 14:30:40.914673", + "local_mtime": "2018-04-05 14:30:40.917705", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 769390837, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, @@ -2998,19 +4476,7 @@ EOF "read_error", "hinfo_missing" ], - "size": 2048, - "attrs": [ - { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - }, - { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - } - ] + "size": 2048 }, { "osd": 1, @@ -3021,23 +4487,7 @@ EOF "hinfo_corrupted" ], "size": 2048, - "attrs": [ - { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - }, - { - "name": "hinfo_key", - "value": "bad-val", - "Base64": false - }, - { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true - } - ] + "hashinfo": "bad-val" }, { "osd": 2, @@ -3047,23 +4497,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x04cfa72f", - "attrs": [ + "hashinfo": { + "cumulative_shard_hashes": [ { - "name": "_", - "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true + "hash": 80717615, + "shard": 0 }, { - "name": "hinfo_key", - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "Base64": true + "hash": 1534491824, + "shard": 1 }, { - "name": "snapset", - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "Base64": true + "hash": 80717615, + "shard": 2 } - ] + ], + "total_chunk_size": 2048 + } } ] }, @@ -3081,7 +4531,40 @@ EOF "union_shard_errors": [ "ec_hash_error" ], - "selected_object_info": "3:21a44c43:::EOBJ7:head(73'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4486.0:1", + "user_version": 10, + "size": 34, + "mtime": "2018-04-05 14:30:50.995009", + "local_mtime": "2018-04-05 14:30:50.996112", + "lost": 0, + "flags": [ + "dirty", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 325996071, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, @@ -3091,7 +4574,23 @@ EOF "ec_hash_error" ], "size": 2048, - "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } }, { "osd": 1, @@ -3101,7 +4600,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x5b7455a8", - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } }, { "osd": 2, @@ -3111,7 +4626,23 @@ EOF "size": 2048, "omap_digest": "0xffffffff", "data_digest": "0x5b7455a8", - "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8" + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } } ] } @@ -3122,7 +4653,7 @@ EOF fi - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then @@ -3306,7 +4837,7 @@ function TEST_corrupt_snapset_scrub_rep() { rados list-inconsistent-obj $pg > $dir/json || return 1 - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "epoch": 34, "inconsistents": [ @@ -3322,21 +4853,78 @@ function TEST_corrupt_snapset_scrub_rep() { "snapset_inconsistency" ], "union_shard_errors": [], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(27'8 client.4143.0:1 dirty|omap|data_digest s 21 uv 8 dd 53acb008 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "24'8", + "prior_version": "21'3", + "last_reqid": "client.4195.0:1", + "user_version": 8, + "size": 21, + "mtime": "2018-04-05 14:35:43.286117", + "local_mtime": "2018-04-05 14:35:43.288990", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 1403826184, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, "primary": false, "errors": [], "size": 21, - "snapset": "1=[1]:{1=[1]}" + "snapset": { + "clones": [ + { + "overlap": "[]", + "size": 7, + "snap": 1, + "snaps": [ + 1 + ] + } + ], + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + } + } }, { "osd": 1, "primary": true, "errors": [], "size": 21, - "snapset": "0=[]:[]+stray_clone_snaps={1=[1]}" + "snapset": { + "clones": [], + "snap_context": { + "seq": 0, + "snaps": [] + } + } } ] }, @@ -3352,21 +4940,78 @@ function TEST_corrupt_snapset_scrub_rep() { "snapset_inconsistency" ], "union_shard_errors": [], - "selected_object_info": "3:e97ce31e:::ROBJ2:head(31'10 client.4155.0:1 dirty|omap|data_digest s 21 uv 10 dd 53acb008 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ2", + "key": "", + "snapid": -2, + "hash": 2026323607, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "28'10", + "prior_version": "23'6", + "last_reqid": "client.4223.0:1", + "user_version": 10, + "size": 21, + "mtime": "2018-04-05 14:35:48.326856", + "local_mtime": "2018-04-05 14:35:48.328097", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": 1403826184, + "omap_digest": 4294967295, + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, "primary": false, "errors": [], "size": 21, - "snapset": "0=[]:[]+stray_clone_snaps={1=[1]}" + "snapset": { + "clones": [], + "snap_context": { + "seq": 0, + "snaps": [] + } + } }, { "osd": 1, "primary": true, "errors": [], "size": 21, - "snapset": "1=[1]:{1=[1]}" + "snapset": { + "clones": [ + { + "overlap": "[]", + "size": 7, + "snap": 1, + "snaps": [ + 1 + ] + } + ], + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + } + } } ] } @@ -3374,7 +5019,7 @@ function TEST_corrupt_snapset_scrub_rep() { } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then diff --git a/src/osd/ECUtil.cc b/src/osd/ECUtil.cc index 3fbfe6e0e278..ae757db7543f 100644 --- a/src/osd/ECUtil.cc +++ b/src/osd/ECUtil.cc @@ -176,7 +176,7 @@ void ECUtil::HashInfo::decode(bufferlist::iterator &bl) void ECUtil::HashInfo::dump(Formatter *f) const { f->dump_unsigned("total_chunk_size", total_chunk_size); - f->open_object_section("cumulative_shard_hashes"); + f->open_array_section("cumulative_shard_hashes"); for (unsigned i = 0; i != cumulative_shard_hashes.size(); ++i) { f->open_object_section("hash"); f->dump_unsigned("shard", i); diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 0fdbb65307c4..14549ba2a6b5 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -5304,7 +5304,11 @@ void object_info_t::dump(Formatter *f) const f->dump_stream("mtime") << mtime; f->dump_stream("local_mtime") << local_mtime; f->dump_unsigned("lost", (int)is_lost()); - f->dump_unsigned("flags", (int)flags); + vector sv = get_flag_vector(flags); + f->open_array_section("flags"); + for (auto str: sv) + f->dump_string("flags", str); + f->close_section(); f->open_array_section("legacy_snaps"); for (auto s : legacy_snaps) { f->dump_unsigned("snap", s); diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 590dcc1eda7a..2d7da93d7a81 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -4671,27 +4671,35 @@ struct object_info_t { static string get_flag_string(flag_t flags) { string s; + vector sv = get_flag_vector(flags); + for (auto ss : sv) { + s += string("|") + ss; + } + if (s.length()) + return s.substr(1); + return s; + } + static vector get_flag_vector(flag_t flags) { + vector sv; if (flags & FLAG_LOST) - s += "|lost"; + sv.insert(sv.end(), "lost"); if (flags & FLAG_WHITEOUT) - s += "|whiteout"; + sv.insert(sv.end(), "whiteout"); if (flags & FLAG_DIRTY) - s += "|dirty"; + sv.insert(sv.end(), "dirty"); if (flags & FLAG_USES_TMAP) - s += "|uses_tmap"; + sv.insert(sv.end(), "uses_tmap"); if (flags & FLAG_OMAP) - s += "|omap"; + sv.insert(sv.end(), "omap"); if (flags & FLAG_DATA_DIGEST) - s += "|data_digest"; + sv.insert(sv.end(), "data_digest"); if (flags & FLAG_OMAP_DIGEST) - s += "|omap_digest"; + sv.insert(sv.end(), "omap_digest"); if (flags & FLAG_CACHE_PIN) - s += "|cache_pin"; + sv.insert(sv.end(), "cache_pin"); if (flags & FLAG_MANIFEST) - s += "|manifest"; - if (s.length()) - return s.substr(1); - return s; + sv.insert(sv.end(), "manifest"); + return sv; } string get_flag_string() const { return get_flag_string(flags); diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index 0e1c66313a6f..0e9d0e89d288 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -1371,47 +1371,73 @@ static void dump_shard(const shard_info_t& shard, f.dump_format("data_digest", "0x%08x", shard.data_digest); } - if (!shard.has_info_missing() && !shard.has_info_corrupted() && - inc.has_object_info_inconsistency()) { - object_info_t oi; - bufferlist bl; + if ((inc.union_shards.has_info_missing() + || inc.union_shards.has_info_corrupted() + || inc.has_object_info_inconsistency() + || shard.has_obj_size_info_mismatch()) && + !shard.has_info_missing()) { map::iterator k = (const_cast(shard)).attrs.find(OI_ATTR); assert(k != shard.attrs.end()); // Can't be missing - bufferlist::iterator bliter = k->second.begin(); - ::decode(oi, bliter); // Can't be corrupted - f.dump_stream("object_info") << oi; + if (!shard.has_info_corrupted()) { + object_info_t oi; + bufferlist bl; + bufferlist::iterator bliter = k->second.begin(); + ::decode(oi, bliter); // Can't be corrupted + f.open_object_section("object_info"); + oi.dump(&f); + f.close_section(); + } else { + bool b64; + f.dump_string("object_info", cleanbin(k->second, b64)); + } } - if (!shard.has_snapset_missing() && !shard.has_snapset_corrupted() && - inc.has_snapset_inconsistency()) { - SnapSet ss; - bufferlist bl; + if ((inc.union_shards.has_snapset_missing() + || inc.union_shards.has_snapset_corrupted() + || inc.has_snapset_inconsistency()) && + !shard.has_snapset_missing()) { map::iterator k = (const_cast(shard)).attrs.find(SS_ATTR); assert(k != shard.attrs.end()); // Can't be missing - bufferlist::iterator bliter = k->second.begin(); - decode(ss, bliter); // Can't be corrupted - f.dump_stream("snapset") << ss; + if (!shard.has_snapset_corrupted()) { + SnapSet ss; + bufferlist bl; + bufferlist::iterator bliter = k->second.begin(); + decode(ss, bliter); // Can't be corrupted + f.open_object_section("snapset"); + ss.dump(&f); + f.close_section(); + } else { + bool b64; + f.dump_string("snapset", cleanbin(k->second, b64)); + } } - if (!shard.has_hinfo_missing() && !shard.has_hinfo_corrupted() && - inc.has_hinfo_inconsistency()) { - ECUtil::HashInfo hi; - bufferlist bl; + if ((inc.union_shards.has_hinfo_missing() + || inc.union_shards.has_hinfo_corrupted() + || inc.has_hinfo_inconsistency()) && + !shard.has_hinfo_missing()) { map::iterator k = (const_cast(shard)).attrs.find(ECUtil::get_hinfo_key()); assert(k != shard.attrs.end()); // Can't be missing - bufferlist::iterator bliter = k->second.begin(); - decode(hi, bliter); // Can't be corrupted - f.dump_stream("hashinfo") << hi; + if (!shard.has_hinfo_corrupted()) { + ECUtil::HashInfo hi; + bufferlist bl; + bufferlist::iterator bliter = k->second.begin(); + decode(hi, bliter); // Can't be corrupted + f.open_object_section("hashinfo"); + hi.dump(&f); + f.close_section(); + } else { + bool b64; + f.dump_string("hashinfo", cleanbin(k->second, b64)); + } } - if (inc.has_attr_name_mismatch() || inc.has_attr_value_mismatch() - || inc.union_shards.has_info_missing() - || inc.union_shards.has_info_corrupted() - || inc.union_shards.has_snapset_missing() - || inc.union_shards.has_snapset_corrupted() - || inc.union_shards.has_hinfo_missing() - || inc.union_shards.has_hinfo_corrupted()) { + if (inc.has_attr_name_mismatch() || inc.has_attr_value_mismatch()) { f.open_array_section("attrs"); for (auto kv : shard.attrs) { + // System attribute handled above + if (kv.first == OI_ATTR || kv.first[0] != '_') + continue; f.open_object_section("attr"); - f.dump_string("name", kv.first); + // Skip leading underscore since only giving user attrs + f.dump_string("name", kv.first.substr(1)); bool b64; f.dump_string("value", cleanbin(kv.second, b64)); f.dump_bool("Base64", b64); @@ -1481,7 +1507,9 @@ static void dump_inconsistent(const inconsistent_obj_t& inc, assert(k != shard.attrs.end()); // Can't be missing bufferlist::iterator bliter = k->second.begin(); ::decode(oi, bliter); // Can't be corrupted - f.dump_stream("selected_object_info") << oi; + f.open_object_section("selected_object_info"); + oi.dump(&f); + f.close_section(); break; } }