osd rados command: Improve scrub handling of HashInfo (hinfo_key xattr)

Fixes: http://tracker.ceph.com/issues/23364
Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/doc/rados/command/list-inconsistent-obj.json b/doc/rados/command/list-inconsistent-obj.json
index 859dc84ca2b5a9aa65ba339e7f8870cd8de54e46..ec2ed0fd170261bcad8c25e1fe8e50d58f460907 100644 (file)
--- a/doc/rados/command/list-inconsistent-obj.json
+++ b/doc/rados/command/list-inconsistent-obj.json
@@ -69,7 +69,9 @@
                 "oi_attr_corrupted",
                 "obj_size_oi_mismatch",
                 "ss_attr_missing",
-                "ss_attr_corrupted"
+                "ss_attr_corrupted",
+                "hinfo_missing",
+                "hinfo_corrupted"
               ]
             },
             "minItems": 0,
@@ -86,7 +88,8 @@
                 "size_mismatch",
                 "attr_value_mismatch",
                 "attr_name_mismatch",
-                "snapset_inconsistency"
+                "snapset_inconsistency",
+                "hinfo_inconsistency"
               ]
             },
             "minItems": 0,
@@ -139,7 +142,9 @@
                       "oi_attr_corrupted",
                       "obj_size_oi_mismatch",
                       "ss_attr_missing",
-                      "ss_attr_corrupted"
+                      "ss_attr_corrupted",
+                      "hinfo_missing",
+                      "hinfo_corrupted"
                     ]
                   },
                   "minItems": 0,

diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh
index 7ba2283cd013a25971c9dde9bcc20fb2d2325ce0..e33445c98478974b54f6988b18aa8982df614a2f 100755 (executable)
--- a/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/qa/standalone/scrub/osd-scrub-repair.sh
@@ -1698,7 +1698,7 @@ function corrupt_scrub_erasure() {
     local dir=$1
     local allow_overwrites=$2
     local poolname=ecpool
-    local total_objs=5
+    local total_objs=7
 
     setup $dir || return 1
     run_mon $dir a || return 1
@@ -1760,6 +1760,23 @@ function corrupt_scrub_erasure() {
             objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
             ;;
 
+        6)
+            objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
+            echo -n bad-val > $dir/bad-val
+            objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
+            ;;
+
+        7)
+            local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
+            echo $payload > $dir/DIFFERENT
+            rados --pool $poolname put $objname $dir/DIFFERENT || return 1
+
+            # Get hinfo_key from EOBJ1
+            objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
+            objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
+            rm -f $dir/hinfo
+            ;;
+
         esac
     done
 
@@ -2022,6 +2039,134 @@ function corrupt_scrub_erasure() {
         "nspace": "",
         "name": "EOBJ5"
       }
+   },
+   {
+     "errors": [],
+     "object": {
+       "locator": "",
+       "name": "EOBJ6",
+       "nspace": "",
+       "snap": "head",
+       "version": 8
+     },
+     "selected_object_info": "3:4e671bad:::EOBJ6:head(65'7 client.4441.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])",
+     "shards": [
+       {
+         "attrs": [
+           {
+             "Base64": true,
+             "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           },
+           {
+             "Base64": true,
+             "name": "snapset",
+             "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           }
+         ],
+         "errors": [
+           "hinfo_missing"
+         ],
+         "osd": 0,
+         "primary": false,
+         "shard": 2,
+         "size": 2048
+       },
+       {
+         "attrs": [
+           {
+             "Base64": true,
+             "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           },
+           {
+             "Base64": false,
+             "name": "hinfo_key",
+             "value": "bad-val"
+           },
+           {
+             "Base64": true,
+             "name": "snapset",
+             "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           }
+         ],
+         "errors": [
+           "hinfo_corrupted"
+         ],
+         "osd": 1,
+         "primary": true,
+         "shard": 0,
+         "size": 2048
+       },
+       {
+         "attrs": [
+           {
+             "Base64": true,
+             "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           },
+           {
+             "Base64": true,
+             "name": "hinfo_key",
+             "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E"
+           },
+           {
+             "Base64": true,
+             "name": "snapset",
+             "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+           }
+         ],
+         "errors": [],
+         "osd": 2,
+         "primary": false,
+         "shard": 1,
+         "size": 2048
+       }
+     ],
+     "union_shard_errors": [
+       "hinfo_missing",
+       "hinfo_corrupted"
+     ]
+   },
+   {
+     "errors": [
+       "hinfo_inconsistency"
+     ],
+     "object": {
+       "locator": "",
+       "name": "EOBJ7",
+       "nspace": "",
+       "snap": "head",
+       "version": 10
+     },
+     "selected_object_info": "3:21a44c43:::EOBJ7:head(65'7 client.4441.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])",
+     "shards": [
+       {
+         "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f",
+         "errors": [],
+         "osd": 0,
+         "primary": false,
+         "shard": 2,
+         "size": 2048
+       },
+       {
+         "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8",
+         "errors": [],
+         "osd": 1,
+         "primary": true,
+         "shard": 0,
+         "size": 2048
+       },
+       {
+         "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8",
+         "errors": [],
+         "osd": 2,
+         "primary": false,
+         "shard": 1,
+         "size": 2048
+       }
+     ],
+     "union_shard_errors": []
     }
   ],
   "epoch": 0
@@ -2321,6 +2466,145 @@ EOF
         "nspace": "",
         "name": "EOBJ5"
       }
+    },
+    {
+      "object": {
+        "name": "EOBJ6",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 8
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "read_error",
+        "hinfo_missing",
+        "hinfo_corrupted"
+      ],
+      "selected_object_info": "3:4e671bad:::EOBJ6:head(65'8 client.4432.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])",
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [
+            "read_error",
+            "hinfo_missing"
+          ],
+          "size": 2048,
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "hinfo_corrupted"
+          ],
+          "size": 2048,
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "hinfo_key",
+              "value": "bad-val",
+              "Base64": false
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAQQAAAAAAAAAAAAAAAAAAAAICFQAAAAhQEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAAV5CqWvL8FjMCAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAABXkKpahjpkM/X42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "hinfo_key",
+              "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
+              "Base64": true
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ7",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 10
+      },
+      "errors": [
+        "hinfo_inconsistency"
+      ],
+      "union_shard_errors": [],
+      "selected_object_info": "3:21a44c43:::EOBJ7:head(75'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])",
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
+          "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f"
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
+          "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8"
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
+          "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8"
+        }
+      ]
     }
   ],
   "epoch": 0
@@ -2639,6 +2923,147 @@ EOF
         "nspace": "",
         "name": "EOBJ5"
       }
+    },
+    {
+      "object": {
+        "name": "EOBJ6",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 8
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "read_error",
+        "hinfo_missing",
+        "hinfo_corrupted"
+      ],
+      "selected_object_info": "3:4e671bad:::EOBJ6:head(63'8 client.4434.0:1 dirty|data_digest s 7 uv 8 dd 2ddbf8f5 alloc_hint [0 0 0])",
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [
+            "read_error",
+            "hinfo_missing"
+          ],
+          "size": 2048,
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "hinfo_corrupted"
+          ],
+          "size": 2048,
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "hinfo_key",
+              "value": "bad-val",
+              "Base64": false
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f",
+          "attrs": [
+            {
+              "name": "_",
+              "value": "EQj+AAAABAMmAAAAAAAAAAUAAABFT0JKNv7/////////cubYtQAAAAAAAwAAAAAAAAAGAxwAAAADAAAAAAAAAP////8AAAAAAAAAAP//////////AAAAAAgAAAAAAAAAPwAAAAAAAAAAAAAAAAAAAAICFQAAAAhSEQAAAAAAAAEAAAAAAAAAAAAAAAcAAAAAAAAA84+qWvTADR4CAhUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAABQAAADzj6pav3EfHvX42y3/////AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            },
+            {
+              "name": "hinfo_key",
+              "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
+              "Base64": true
+            },
+            {
+              "name": "snapset",
+              "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+              "Base64": true
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ7",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 10
+      },
+      "errors": [
+        "hinfo_inconsistency"
+      ],
+      "union_shard_errors": [
+        "ec_hash_error"
+      ],
+      "selected_object_info": "3:21a44c43:::EOBJ7:head(73'10 client.4498.0:1 dirty|data_digest s 34 uv 10 dd 136e4e27 alloc_hint [0 0 0])",
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [
+            "ec_hash_error"
+          ],
+          "size": 2048,
+          "hashinfo": "tcs=2048 4cfa72f 5b767cb0 4cfa72f"
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x5b7455a8",
+          "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8"
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x5b7455a8",
+          "hashinfo": "tcs=2048 5b7455a8 5b767cb0 5b7455a8"
+        }
+      ]
     }
   ],
   "epoch": 0

diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h
index 2a17928344a374244e173d73dfcacf6e8b0f8329..4285c4637e56d8678f95b36d8e09f722dd5c85f5 100644 (file)
--- a/src/common/scrub_types.h
+++ b/src/common/scrub_types.h
@@ -83,6 +83,12 @@ public:
   void set_obj_size_oi_mismatch() {
     errors |= err_t::OBJ_SIZE_OI_MISMATCH;
   }
+  void set_hinfo_missing() {
+    errors |= err_t::HINFO_MISSING;
+  }
+  void set_hinfo_corrupted() {
+    errors |= err_t::HINFO_CORRUPTED;
+  }
   void encode(bufferlist& bl) const;
   void decode(bufferlist::iterator& bp);
 };
@@ -120,6 +126,9 @@ struct inconsistent_obj_wrapper : librados::inconsistent_obj_t {
   void set_snapset_inconsistency() {
     errors |= obj_err_t::SNAPSET_INCONSISTENCY;
   }
+  void set_hinfo_inconsistency() {
+    errors |= obj_err_t::HINFO_INCONSISTENCY;
+  }
   void add_shard(const pg_shard_t& pgs, const shard_info_wrapper& shard);
   void set_auth_missing(const hobject_t& hoid,
                         const map<pg_shard_t, ScrubMap*>&,

diff --git a/src/include/rados/rados_types.hpp b/src/include/rados/rados_types.hpp
index 37e9d285459882fd499980ef92899348a0688024..7d8d075e1ff93900a3f0afbd92e025316af4f849 100644 (file)
--- a/src/include/rados/rados_types.hpp
+++ b/src/include/rados/rados_types.hpp
@@ -64,11 +64,13 @@ struct err_t {
     OI_ATTR_CORRUPTED       = 1 << 15,
     SS_ATTR_MISSING         = 1 << 16,
     SS_ATTR_CORRUPTED       = 1 << 17,
-    OBJ_SIZE_OI_MISMATCH      = 1 << 18
+    OBJ_SIZE_OI_MISMATCH      = 1 << 18,
+    HINFO_MISSING         = 1 << 19,
+    HINFO_CORRUPTED       = 1 << 20
     // When adding more here add to either SHALLOW_ERRORS or DEEP_ERRORS
   };
   uint64_t errors = 0;
-  static constexpr uint64_t SHALLOW_ERRORS = SHARD_MISSING|SHARD_STAT_ERR|SIZE_MISMATCH_OI|OI_ATTR_MISSING|OI_ATTR_CORRUPTED|SS_ATTR_MISSING|SS_ATTR_CORRUPTED|OBJ_SIZE_OI_MISMATCH;
+  static constexpr uint64_t SHALLOW_ERRORS = SHARD_MISSING|SHARD_STAT_ERR|SIZE_MISMATCH_OI|OI_ATTR_MISSING|OI_ATTR_CORRUPTED|SS_ATTR_MISSING|SS_ATTR_CORRUPTED|OBJ_SIZE_OI_MISMATCH|HINFO_MISSING|HINFO_CORRUPTED;
   static constexpr uint64_t DEEP_ERRORS = SHARD_READ_ERR|DATA_DIGEST_MISMATCH_OI|OMAP_DIGEST_MISMATCH_OI|SHARD_EC_HASH_MISMATCH|SHARD_EC_SIZE_MISMATCH;
   bool has_shard_missing() const {
     return errors & SHARD_MISSING;
@@ -115,6 +117,12 @@ struct err_t {
   bool has_obj_size_oi_mismatch() const {
     return errors & OBJ_SIZE_OI_MISMATCH;
   }
+  bool has_hinfo_missing() const {
+    return errors & HINFO_MISSING;
+  }
+  bool has_hinfo_corrupted() const {
+    return errors & HINFO_CORRUPTED;
+  }
 };
 
 struct shard_info_t : err_t {
@@ -152,10 +160,11 @@ struct obj_err_t {
     ATTR_VALUE_MISMATCH  = 1 << 7,
     ATTR_NAME_MISMATCH    = 1 << 8,
     SNAPSET_INCONSISTENCY   = 1 << 9,
+    HINFO_INCONSISTENCY   = 1 << 10,
     // When adding more here add to either SHALLOW_ERRORS or DEEP_ERRORS
   };
   uint64_t errors = 0;
-  static constexpr uint64_t SHALLOW_ERRORS = OBJECT_INFO_INCONSISTENCY|SIZE_MISMATCH|ATTR_VALUE_MISMATCH|ATTR_NAME_MISMATCH|SNAPSET_INCONSISTENCY;
+  static constexpr uint64_t SHALLOW_ERRORS = OBJECT_INFO_INCONSISTENCY|SIZE_MISMATCH|ATTR_VALUE_MISMATCH|ATTR_NAME_MISMATCH|SNAPSET_INCONSISTENCY|HINFO_INCONSISTENCY;
   static constexpr uint64_t DEEP_ERRORS = DATA_DIGEST_MISMATCH|OMAP_DIGEST_MISMATCH;
   bool has_object_info_inconsistency() const {
     return errors & OBJECT_INFO_INCONSISTENCY;
@@ -184,6 +193,9 @@ struct obj_err_t {
   bool has_snapset_inconsistency() const {
     return errors & SNAPSET_INCONSISTENCY;
   }
+  bool has_hinfo_inconsistency() const {
+    return errors & HINFO_INCONSISTENCY;
+  }
 };
 
 struct inconsistent_obj_t : obj_err_t {

diff --git a/src/osd/ECUtil.cc b/src/osd/ECUtil.cc
index 6d538f0003404d888d90bdd9aea9eac19d2c55ea..a6704953fc75f714713d8c924f86fe8fd0815645 100644 (file)
--- a/src/osd/ECUtil.cc
+++ b/src/osd/ECUtil.cc
@@ -206,6 +206,16 @@ void ECUtil::HashInfo::dump(Formatter *f) const
   f->close_section();
 }
 
+namespace ECUtil {
+std::ostream& operator<<(std::ostream& out, const HashInfo& hi)
+{
+  ostringstream hashes;
+  for (auto hash: hi.cumulative_shard_hashes)
+    hashes << " " << hex << hash;
+  return out << "tcs=" << hi.total_chunk_size << hashes.str();
+}
+}
+
 void ECUtil::HashInfo::generate_test_instances(list<HashInfo*>& o)
 {
   o.push_back(new HashInfo(3));

diff --git a/src/osd/ECUtil.h b/src/osd/ECUtil.h
index 94ce1a1a4d195652931bebe63f8b68a91f7bceba..fb2e8ad283c39edb2ff235d6de5ae7bec4871b66 100644 (file)
--- a/src/osd/ECUtil.h
+++ b/src/osd/ECUtil.h
@@ -15,6 +15,7 @@
 #ifndef ECUTIL_H
 #define ECUTIL_H
 
+#include <ostream>
 #include "erasure-code/ErasureCodeInterface.h"
 #include "include/buffer_fwd.h"
 #include "include/assert.h"
@@ -155,6 +156,7 @@ public:
     *this = rhs;
     projected_total_chunk_size = ptcs;
   }
+  friend std::ostream& operator<<(std::ostream& out, const HashInfo& hi);
 };
 
 typedef ceph::shared_ptr<HashInfo> HashInfoRef;

diff --git a/src/osd/PGBackend.cc b/src/osd/PGBackend.cc
index bd4d98681c893f600cf91e7f82db4ff791d83ffa..dde1f3965d85d014986c4c763b65c744579723b1 100644 (file)
--- a/src/osd/PGBackend.cc
+++ b/src/osd/PGBackend.cc
@@ -707,7 +707,7 @@ bool PGBackend::be_compare_scrub_objects(
        i != auth.attrs.end();
        ++i) {
     // We check system keys seperately
-    if (i->first == OI_ATTR || i->first == SS_ATTR)
+    if (i->first == OI_ATTR || i->first[0] != '_')
       continue;
     if (!candidate.attrs.count(i->first)) {
       if (error != CLEAN)
@@ -727,7 +727,7 @@ bool PGBackend::be_compare_scrub_objects(
        i != candidate.attrs.end();
        ++i) {
     // We check system keys seperately
-    if (i->first == OI_ATTR || i->first == SS_ATTR)
+    if (i->first == OI_ATTR || i->first[0] != '_')
       continue;
     if (!auth.attrs.count(i->first)) {
       if (error != CLEAN)
@@ -759,7 +759,7 @@ map<pg_shard_t, ScrubMap *>::const_iterator
   inconsistent_obj_wrapper &object_error)
 {
   eversion_t auth_version;
-  bufferlist first_oi_bl, first_ss_bl;
+  bufferlist first_oi_bl, first_ss_bl, first_hk_bl;
 
   // Create list of shards with primary first so it will be auth copy all
   // other things being equal.
@@ -802,7 +802,7 @@ map<pg_shard_t, ScrubMap *>::const_iterator
     bufferlist bl;
     map<string, bufferptr>::iterator k;
     SnapSet ss;
-    bufferlist ss_bl;
+    bufferlist ss_bl, hk_bl;
 
     if (i->second.stat_error) {
       shard_info.set_stat_error();
@@ -838,6 +838,31 @@ map<pg_shard_t, ScrubMap *>::const_iterator
       }
     }
 
+    if (parent->get_pool().is_erasure()) {
+      ECUtil::HashInfo hi;
+      k = i->second.attrs.find(ECUtil::get_hinfo_key());
+      if (k == i->second.attrs.end()) {
+       shard_info.set_hinfo_missing();
+       error_string += " hinfo_key_missing";
+      } else {
+       hk_bl.push_back(k->second);
+        try {
+         bufferlist::iterator bliter = hk_bl.begin();
+         decode(hi, bliter);
+         if (first_hk_bl.length() == 0) {
+           first_hk_bl.append(hk_bl);
+         } else if (!object_error.has_hinfo_inconsistency() && !hk_bl.contents_equal(first_hk_bl)) {
+           object_error.set_hinfo_inconsistency();
+           error_string += " hinfo_inconsistency";
+         }
+        } catch (...) {
+         // invalid snapset, probably corrupt
+         shard_info.set_hinfo_corrupted();
+         error_string += " hinfo_corrupted";
+        }
+      }
+    }
+
     k = i->second.attrs.find(OI_ATTR);
     if (k == i->second.attrs.end()) {
       // no object info on object, probably corrupt

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index 6e30ed2e6d973cc71e57b357bf74382a1005ebf9..e0844ec0c7a1f56928ba383e06264ec6450b86d6 100644 (file)
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -4,7 +4,8 @@ set(rados_srcs
   rados/RadosImport.cc
   rados/PoolDump.cc
   ${PROJECT_SOURCE_DIR}/src/common/util.cc
-  ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc)
+  ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc)
 add_executable(rados ${rados_srcs})
 target_link_libraries(rados librados global ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS} radosstriper)
 install(TARGETS rados DESTINATION bin)

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 4fd2d696508eb08b20f084aea8699c278b1f78c9..a9ebc4efea8b4e309265251e92752f176b032a9b 100644 (file)
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -52,6 +52,8 @@ using namespace libradosstriper;
 #include "PoolDump.h"
 #include "RadosImport.h"
 
+#include "osd/ECUtil.h"
+
 using namespace librados;
 using ceph::util::generate_random_number;
 
@@ -1348,6 +1350,10 @@ static void dump_errors(const err_t &err, Formatter &f, const char *name)
     f.dump_string("error", "ss_attr_missing");
   if (err.has_ss_attr_corrupted())
     f.dump_string("error", "ss_attr_corrupted");
+  if (err.has_hinfo_missing())
+    f.dump_string("error", "hinfo_missing");
+  if (err.has_hinfo_corrupted())
+    f.dump_string("error", "hinfo_corrupted");
   f.close_section();
 }
 
@@ -1389,11 +1395,23 @@ static void dump_shard(const shard_info_t& shard,
     decode(ss, bliter);  // Can't be corrupted
     f.dump_stream("snapset") << ss;
   }
+  if (!shard.has_hinfo_missing() && !shard.has_hinfo_corrupted() &&
+      inc.has_hinfo_inconsistency()) {
+    ECUtil::HashInfo hi;
+    bufferlist bl;
+    map<std::string, ceph::bufferlist>::iterator k = (const_cast<shard_info_t&>(shard)).attrs.find(ECUtil::get_hinfo_key());
+    assert(k != shard.attrs.end()); // Can't be missing
+    bufferlist::iterator bliter = k->second.begin();
+    decode(hi, bliter);  // Can't be corrupted
+    f.dump_stream("hashinfo") << hi;
+  }
   if (inc.has_attr_name_mismatch() || inc.has_attr_value_mismatch()
      || inc.union_shards.has_oi_attr_missing()
      || inc.union_shards.has_oi_attr_corrupted()
      || inc.union_shards.has_ss_attr_missing()
-     || inc.union_shards.has_ss_attr_corrupted()) {
+     || inc.union_shards.has_ss_attr_corrupted()
+     || inc.union_shards.has_hinfo_missing()
+     || inc.union_shards.has_hinfo_corrupted()) {
     f.open_array_section("attrs");
     for (auto kv : shard.attrs) {
       f.open_object_section("attr");
@@ -1424,6 +1442,8 @@ static void dump_obj_errors(const obj_err_t &err, Formatter &f)
     f.dump_string("error", "attr_name_mismatch");
   if (err.has_snapset_inconsistency())
     f.dump_string("error", "snapset_inconsistency");
+  if (err.has_hinfo_inconsistency())
+    f.dump_string("error", "hinfo_inconsistency");
   f.close_section();
 }