From 3aceb1fd44876877a6ddca7cd14eb3c2f8ed8790 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 28 Apr 2025 08:14:54 -0500
Subject: [PATCH] qa/standalone/scrub: fix expected output in EC repair tests

Specifically - TEST_corrupt_scrub_erasure_* in osd-scrub-repair.sh.
This is required following recent changes to the EC code.

Fixes: https://tracker.ceph.com/issues/70851
Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/scrub/osd-scrub-repair.sh | 2008 ++++++++++++-----------
 1 file changed, 1017 insertions(+), 991 deletions(-)

diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh
index 901a64da8ccde..3ec7d421342dd 100755
--- a/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/qa/standalone/scrub/osd-scrub-repair.sh
@@ -3748,17 +3748,75 @@ function corrupt_scrub_erasure() {
 
     jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
 {
+  "epoch": 99,
   "inconsistents": [
     {
+      "object": {
+        "name": "EOBJ1",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 1
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
+      ],
+      "selected_object_info": {
+        "oid": {
+          "oid": "EOBJ1",
+          "key": "",
+          "snapid": -2,
+          "hash": 560836233,
+          "max": 0,
+          "pool": 3,
+          "namespace": ""
+        },
+        "version": "32'1",
+        "prior_version": "0'0",
+        "last_reqid": "client.4210.0:1",
+        "user_version": 1,
+        "size": 7,
+        "mtime": "2025-04-28T02:18:19.605985-0500",
+        "local_mtime": "2025-04-28T02:18:19.607916-0500",
+        "lost": 0,
+        "flags": [
+          "dirty",
+          "data_digest"
+        ],
+        "truncate_seq": 0,
+        "truncate_size": 0,
+        "data_digest": "0x2ddbf8f5",
+        "omap_digest": "0xffffffff",
+        "expected_object_size": 0,
+        "expected_write_size": 0,
+        "alloc_hint_flags": 0,
+        "manifest": {
+          "type": 0
+        },
+        "watchers": {},
+        "shard_versions": []
+      },
       "shards": [
         {
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 9,
           "object_info": {
             "oid": {
               "oid": "EOBJ1",
@@ -3769,13 +3827,13 @@ function corrupt_scrub_erasure() {
               "pool": 3,
               "namespace": ""
             },
-            "version": "27'1",
+            "version": "32'1",
             "prior_version": "0'0",
-            "last_reqid": "client.4184.0:1",
+            "last_reqid": "client.4210.0:1",
             "user_version": 1,
             "size": 7,
-            "mtime": "",
-            "local_mtime": "",
+            "mtime": "2025-04-28T02:18:19.605985-0500",
+            "local_mtime": "2025-04-28T02:18:19.607916-0500",
             "lost": 0,
             "flags": [
               "dirty",
@@ -3791,42 +3849,48 @@ function corrupt_scrub_erasure() {
             "manifest": {
               "type": 0
             },
-            "watchers": {}
-          },
-          "size": 9,
-          "shard": 0,
-          "errors": [
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "osd": 1,
-          "primary": true
+            "watchers": {},
+            "shard_versions": []
+          }
         },
         {
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ3",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 3
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "missing"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ1",
+          "oid": "EOBJ3",
           "key": "",
           "snapid": -2,
-          "hash": 560836233,
+          "hash": 3125668237,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "27'1",
+        "version": "46'3",
         "prior_version": "0'0",
-        "last_reqid": "client.4184.0:1",
-        "user_version": 1,
+        "last_reqid": "client.4286.0:1",
+        "user_version": 3,
         "size": 7,
-        "mtime": "",
-        "local_mtime": "",
+        "mtime": "2025-04-28T02:18:36.149933-0500",
+        "local_mtime": "2025-04-28T02:18:36.155056-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -3842,65 +3906,64 @@ function corrupt_scrub_erasure() {
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
-      ],
-      "object": {
-        "version": 1,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ1"
-      }
-    },
-    {
       "shards": [
         {
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048
         },
         {
+          "osd": 1,
+          "primary": true,
           "shard": 0,
           "errors": [
             "missing"
-          ],
-          "osd": 1,
-          "primary": true
+          ]
         },
         {
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ4",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 6
+      },
+      "errors": [
+        "attr_value_mismatch",
+        "attr_name_mismatch"
       ],
+      "union_shard_errors": [],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ3",
+          "oid": "EOBJ4",
           "key": "",
           "snapid": -2,
-          "hash": 3125668237,
+          "hash": 1618759290,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "39'3",
-        "prior_version": "0'0",
-        "last_reqid": "client.4252.0:1",
-        "user_version": 3,
+        "version": "54'6",
+        "prior_version": "54'5",
+        "last_reqid": "client.4330.0:1",
+        "user_version": 6,
         "size": 7,
-        "mtime": "",
-        "local_mtime": "",
+        "mtime": "2025-04-28T02:18:44.594720-0500",
+        "local_mtime": "2025-04-28T02:18:44.596235-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -3916,40 +3979,28 @@ function corrupt_scrub_erasure() {
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [
-        "missing"
-      ],
-      "errors": [],
-      "object": {
-        "version": 3,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ3"
-      }
-    },
-    {
       "shards": [
         {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "bad-val",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
-          ],
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
-          "osd": 0,
-          "primary": false
+          ]
         },
         {
           "osd": 1,
@@ -3959,14 +4010,14 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
           ]
         },
@@ -3978,35 +4029,51 @@ function corrupt_scrub_erasure() {
           "size": 2048,
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key3-EOBJ4",
               "value": "val3-EOBJ4",
-              "name": "key3-EOBJ4"
+              "Base64": false
             }
           ]
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ5",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 7
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ4",
+          "oid": "EOBJ5",
           "key": "",
           "snapid": -2,
-          "hash": 1618759290,
+          "hash": 2918945441,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "45'6",
-        "prior_version": "45'5",
-        "last_reqid": "client.4294.0:1",
-        "user_version": 6,
+        "version": "71'7",
+        "prior_version": "0'0",
+        "last_reqid": "client.4424.0:1",
+        "user_version": 7,
         "size": 7,
-        "mtime": "",
-        "local_mtime": "",
+        "mtime": "2025-04-28T02:19:03.683994-0500",
+        "local_mtime": "2025-04-28T02:19:03.685505-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4022,31 +4089,26 @@ function corrupt_scrub_erasure() {
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [],
-      "errors": [
-        "attr_value_mismatch",
-        "attr_name_mismatch"
-      ],
-      "object": {
-        "version": 6,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ4"
-      }
-    },
-    {
       "shards": [
         {
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 4096,
           "object_info": {
             "oid": {
               "oid": "EOBJ5",
@@ -4057,13 +4119,13 @@ function corrupt_scrub_erasure() {
               "pool": 3,
               "namespace": ""
             },
-            "version": "59'7",
+            "version": "71'7",
             "prior_version": "0'0",
-            "last_reqid": "client.4382.0:1",
+            "last_reqid": "client.4424.0:1",
             "user_version": 7,
             "size": 7,
-            "mtime": "",
-            "local_mtime": "",
+            "mtime": "2025-04-28T02:19:03.683994-0500",
+            "local_mtime": "2025-04-28T02:19:03.685505-0500",
             "lost": 0,
             "flags": [
               "dirty",
@@ -4079,84 +4141,33 @@ function corrupt_scrub_erasure() {
             "manifest": {
               "type": 0
             },
-            "watchers": {}
-          },
-          "size": 4096,
-          "shard": 0,
-          "errors": [
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "osd": 1,
-          "primary": true
+            "watchers": {},
+            "shard_versions": []
+          }
         },
         {
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048
         }
-      ],
-      "selected_object_info": {
-        "oid": {
-          "oid": "EOBJ5",
-          "key": "",
-          "snapid": -2,
-          "hash": 2918945441,
-          "max": 0,
-          "pool": 3,
-          "namespace": ""
-        },
-        "version": "59'7",
-        "prior_version": "0'0",
-        "last_reqid": "client.4382.0:1",
-        "user_version": 7,
-        "size": 7,
-        "mtime": "",
-        "local_mtime": "",
-        "lost": 0,
-        "flags": [
-          "dirty",
-          "data_digest"
-        ],
-        "truncate_seq": 0,
-        "truncate_size": 0,
-        "data_digest": "0x2ddbf8f5",
-        "omap_digest": "0xffffffff",
-        "expected_object_size": 0,
-        "expected_write_size": 0,
-        "alloc_hint_flags": 0,
-        "manifest": {
-          "type": 0
-        },
-        "watchers": {}
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ6",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 8
       },
+      "errors": [],
       "union_shard_errors": [
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
+        "hinfo_missing",
+        "hinfo_corrupted"
       ],
-      "object": {
-        "version": 7,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ5"
-      }
-   },
-   {
-     "errors": [],
-     "object": {
-       "locator": "",
-       "name": "EOBJ6",
-       "nspace": "",
-       "snap": "head",
-       "version": 8
-     },
-     "selected_object_info": {
+      "selected_object_info": {
         "oid": {
           "oid": "EOBJ6",
           "key": "",
@@ -4166,13 +4177,13 @@ function corrupt_scrub_erasure() {
           "pool": 3,
           "namespace": ""
         },
-        "version": "65'8",
+        "version": "78'8",
         "prior_version": "0'0",
-        "last_reqid": "client.4418.0:1",
+        "last_reqid": "client.4462.0:1",
         "user_version": 8,
         "size": 7,
-        "mtime": "",
-        "local_mtime": "",
+        "mtime": "2025-04-28T02:19:11.980798-0500",
+        "local_mtime": "2025-04-28T02:19:11.986446-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4188,70 +4199,68 @@ function corrupt_scrub_erasure() {
         "manifest": {
           "type": 0
         },
-        "watchers": {}
-     },
-     "shards": [
-       {
-         "errors": [
-           "hinfo_missing"
-         ],
-         "osd": 0,
-         "primary": false,
-         "shard": 2,
-         "size": 2048
-       },
-       {
-         "errors": [
-           "hinfo_corrupted"
-         ],
-         "osd": 1,
-         "primary": true,
-         "shard": 0,
-         "hashinfo": "bad-val",
-         "size": 2048
-       },
-       {
-         "errors": [],
-         "osd": 2,
-         "primary": false,
-         "shard": 1,
-         "size": 2048,
-         "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         }
-       }
-     ],
-     "union_shard_errors": [
-       "hinfo_missing",
-       "hinfo_corrupted"
-     ]
-   },
-   {
-     "errors": [
-       "hinfo_inconsistency"
-     ],
-     "object": {
-       "locator": "",
-       "name": "EOBJ7",
-       "nspace": "",
-       "snap": "head",
-       "version": 10
-     },
-     "selected_object_info": {
+        "watchers": {},
+        "shard_versions": []
+      },
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [
+            "hinfo_missing"
+          ],
+          "size": 2048
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "hinfo_corrupted"
+          ],
+          "size": 2048,
+          "hashinfo": "bad-val"
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "hashinfo": {
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
+          }
+        }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ7",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 10
+      },
+      "errors": [
+        "hinfo_inconsistency"
+      ],
+      "union_shard_errors": [],
+      "selected_object_info": {
         "oid": {
           "oid": "EOBJ7",
           "key": "",
@@ -4261,13 +4270,13 @@ function corrupt_scrub_erasure() {
           "pool": 3,
           "namespace": ""
         },
-        "version": "75'10",
-        "prior_version": "75'9",
-        "last_reqid": "client.4482.0:1",
+        "version": "90'10",
+        "prior_version": "90'9",
+        "last_reqid": "client.4534.0:1",
         "user_version": 10,
         "size": 34,
-        "mtime": "",
-        "local_mtime": "",
+        "mtime": "2025-04-28T02:19:27.775012-0500",
+        "local_mtime": "2025-04-28T02:19:27.776394-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4283,86 +4292,85 @@ function corrupt_scrub_erasure() {
         "manifest": {
           "type": 0
         },
-        "watchers": {}
-     },
-     "shards": [
-       {
-         "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         },
-         "errors": [],
-         "osd": 0,
-         "primary": false,
-         "shard": 2,
-         "size": 2048
-       },
-       {
-         "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         },
-         "errors": [],
-         "osd": 1,
-         "primary": true,
-         "shard": 0,
-         "size": 2048
-       },
-       {
-         "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         },
-         "errors": [],
-         "osd": 2,
-         "primary": false,
-         "shard": 1,
-         "size": 2048
-       }
-     ],
-     "union_shard_errors": []
+        "watchers": {},
+        "shard_versions": []
+      },
+      "shards": [
+        {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "hashinfo": {
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
+          }
+        },
+        {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [],
+          "size": 2048,
+          "hashinfo": {
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
+          }
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "hashinfo": {
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
+          }
+        }
+      ]
     }
-  ],
-  "epoch": 0
+  ]
 }
 EOF
 
@@ -4387,6 +4395,7 @@ EOF
     test $(jq -r '.[0]' $dir/json) = $pg || return 1
 
     rados list-inconsistent-obj $pg > $dir/json || return 1
+    cp $dir/json /tmp/rrr6.json
     # Get epoch for repair-get requests
     epoch=$(jq .epoch $dir/json)
 
@@ -4394,19 +4403,79 @@ EOF
     then
       jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
 {
+  "epoch": 99,
   "inconsistents": [
     {
+      "object": {
+        "name": "EOBJ1",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 1
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "read_error",
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
+      ],
+      "selected_object_info": {
+        "oid": {
+          "oid": "EOBJ1",
+          "key": "",
+          "snapid": -2,
+          "hash": 560836233,
+          "max": 0,
+          "pool": 3,
+          "namespace": ""
+        },
+        "version": "33'1",
+        "prior_version": "0'0",
+        "last_reqid": "client.4212.0:1",
+        "user_version": 1,
+        "size": 7,
+        "mtime": "2025-04-28T05:22:43.385320-0500",
+        "local_mtime": "2025-04-28T05:22:43.387170-0500",
+        "lost": 0,
+        "flags": [
+          "dirty",
+          "data_digest"
+        ],
+        "truncate_seq": 0,
+        "truncate_size": 0,
+        "data_digest": "0x2ddbf8f5",
+        "omap_digest": "0xffffffff",
+        "expected_object_size": 0,
+        "expected_write_size": 0,
+        "alloc_hint_flags": 0,
+        "manifest": {
+          "type": 0
+        },
+        "watchers": {},
+        "shard_versions": []
+      },
       "shards": [
         {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 9,
           "object_info": {
             "oid": {
               "oid": "EOBJ1",
@@ -4417,13 +4486,13 @@ EOF
               "pool": 3,
               "namespace": ""
             },
-            "version": "27'1",
+            "version": "33'1",
             "prior_version": "0'0",
-            "last_reqid": "client.4184.0:1",
+            "last_reqid": "client.4212.0:1",
             "user_version": 1,
             "size": 7,
-            "mtime": "2018-04-05 14:31:33.837147",
-            "local_mtime": "2018-04-05 14:31:33.840763",
+            "mtime": "2025-04-28T05:22:43.385320-0500",
+            "local_mtime": "2025-04-28T05:22:43.387170-0500",
             "lost": 0,
             "flags": [
               "dirty",
@@ -4439,124 +4508,127 @@ EOF
             "manifest": {
               "type": 0
             },
-            "watchers": {}
-          },
-          "size": 9,
-          "shard": 0,
-          "errors": [
-            "read_error",
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "osd": 1,
-          "primary": true
+            "watchers": {},
+            "shard_versions": []
+          }
         },
         {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ3",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 3
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "missing"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ1",
+          "oid": "EOBJ3",
           "key": "",
           "snapid": -2,
-          "hash": 560836233,
+          "hash": 3125668237,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "27'1",
+        "version": "47'3",
         "prior_version": "0'0",
-        "last_reqid": "client.4184.0:1",
-        "user_version": 1,
+        "last_reqid": "client.4288.0:1",
+        "user_version": 3,
         "size": 7,
-        "mtime": "2018-04-05 14:31:33.837147",
-        "local_mtime": "2018-04-05 14:31:33.840763",
+        "mtime": "2025-04-28T05:22:59.744067-0500",
+        "local_mtime": "2025-04-28T05:22:59.748636-0500",
         "lost": 0,
         "flags": [
           "dirty",
           "data_digest"
-        ],
-        "truncate_seq": 0,
-        "truncate_size": 0,
-        "data_digest": "0x2ddbf8f5",
-        "omap_digest": "0xffffffff",
-        "expected_object_size": 0,
-        "expected_write_size": 0,
-        "alloc_hint_flags": 0,
-        "manifest": {
-          "type": 0
-        },
-        "watchers": {}
-      },
-      "union_shard_errors": [
-        "read_error",
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
-      ],
-      "object": {
-        "version": 1,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ1"
-      }
-    },
-    {
+        ],
+        "truncate_seq": 0,
+        "truncate_size": 0,
+        "data_digest": "0x2ddbf8f5",
+        "omap_digest": "0xffffffff",
+        "expected_object_size": 0,
+        "expected_write_size": 0,
+        "alloc_hint_flags": 0,
+        "manifest": {
+          "type": 0
+        },
+        "watchers": {},
+        "shard_versions": []
+      },
       "shards": [
         {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
         },
         {
+          "osd": 1,
+          "primary": true,
           "shard": 0,
           "errors": [
             "missing"
-          ],
-          "osd": 1,
-          "primary": true
+          ]
         },
         {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ4",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 6
+      },
+      "errors": [
+        "attr_value_mismatch",
+        "attr_name_mismatch"
       ],
+      "union_shard_errors": [],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ3",
+          "oid": "EOBJ4",
           "key": "",
           "snapid": -2,
-          "hash": 3125668237,
+          "hash": 1618759290,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "39'3",
-        "prior_version": "0'0",
-        "last_reqid": "client.4252.0:1",
-        "user_version": 3,
+        "version": "54'6",
+        "prior_version": "54'5",
+        "last_reqid": "client.4332.0:1",
+        "user_version": 6,
         "size": 7,
-        "mtime": "2018-04-05 14:31:46.841145",
-        "local_mtime": "2018-04-05 14:31:46.844996",
+        "mtime": "2025-04-28T05:23:07.959787-0500",
+        "local_mtime": "2025-04-28T05:23:07.961221-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4572,103 +4644,108 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [
-        "missing"
-      ],
-      "errors": [],
-      "object": {
-        "version": 3,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ3"
-      }
-    },
-    {
       "shards": [
         {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "bad-val",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
-          ],
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
-          "osd": 0,
-          "primary": false
+          ]
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
-          ],
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 0,
-          "osd": 1,
-          "primary": true
+          ]
         },
         {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key3-EOBJ4",
               "value": "val3-EOBJ4",
-              "name": "key3-EOBJ4"
+              "Base64": false
             }
-          ],
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 1,
-          "osd": 2,
-          "primary": false
+          ]
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ5",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 7
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "read_error",
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ4",
+          "oid": "EOBJ5",
           "key": "",
           "snapid": -2,
-          "hash": 1618759290,
+          "hash": 2918945441,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "45'6",
-        "prior_version": "45'5",
-        "last_reqid": "client.4294.0:1",
-        "user_version": 6,
+        "version": "71'7",
+        "prior_version": "0'0",
+        "last_reqid": "client.4432.0:1",
+        "user_version": 7,
         "size": 7,
-        "mtime": "2018-04-05 14:31:54.663622",
-        "local_mtime": "2018-04-05 14:31:54.664527",
+        "mtime": "2025-04-28T05:23:31.119695-0500",
+        "local_mtime": "2025-04-28T05:23:31.121349-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4684,33 +4761,29 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [],
-      "errors": [
-        "attr_value_mismatch",
-        "attr_name_mismatch"
-      ],
-      "object": {
-        "version": 6,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ4"
-      }
-    },
-    {
       "shards": [
         {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 4096,
           "object_info": {
             "oid": {
               "oid": "EOBJ5",
@@ -4721,99 +4794,42 @@ EOF
               "pool": 3,
               "namespace": ""
             },
-            "version": "59'7",
+            "version": "71'7",
             "prior_version": "0'0",
-            "last_reqid": "client.4382.0:1",
+            "last_reqid": "client.4432.0:1",
             "user_version": 7,
             "size": 7,
-            "mtime": "2018-04-05 14:32:12.929161",
-            "local_mtime": "2018-04-05 14:32:12.934707",
+            "mtime": "2025-04-28T05:23:31.119695-0500",
+            "local_mtime": "2025-04-28T05:23:31.121349-0500",
             "lost": 0,
             "flags": [
               "dirty",
               "data_digest"
-            ],
-            "truncate_seq": 0,
-            "truncate_size": 0,
-            "data_digest": "0x2ddbf8f5",
-            "omap_digest": "0xffffffff",
-            "expected_object_size": 0,
-            "expected_write_size": 0,
-            "alloc_hint_flags": 0,
-            "manifest": {
-              "type": 0
-            },
-            "watchers": {}
-          },
-          "size": 4096,
-          "errors": [
-            "read_error",
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "shard": 0,
-          "osd": 1,
-          "primary": true
-        },
-        {
-          "data_digest": "0x00000000",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 1,
-          "osd": 2,
-          "primary": false
-        }
-      ],
-      "selected_object_info": {
-        "oid": {
-          "oid": "EOBJ5",
-          "key": "",
-          "snapid": -2,
-          "hash": 2918945441,
-          "max": 0,
-          "pool": 3,
-          "namespace": ""
-        },
-        "version": "59'7",
-        "prior_version": "0'0",
-        "last_reqid": "client.4382.0:1",
-        "user_version": 7,
-        "size": 7,
-        "mtime": "2018-04-05 14:32:12.929161",
-        "local_mtime": "2018-04-05 14:32:12.934707",
-        "lost": 0,
-        "flags": [
-          "dirty",
-          "data_digest"
-        ],
-        "truncate_seq": 0,
-        "truncate_size": 0,
-        "data_digest": "0x2ddbf8f5",
-        "omap_digest": "0xffffffff",
-        "expected_object_size": 0,
-        "expected_write_size": 0,
-        "alloc_hint_flags": 0,
-        "manifest": {
-          "type": 0
-        },
-        "watchers": {}
-      },
-      "union_shard_errors": [
-        "read_error",
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
-      ],
-      "object": {
-        "version": 7,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ5"
-      }
+            ],
+            "truncate_seq": 0,
+            "truncate_size": 0,
+            "data_digest": "0x2ddbf8f5",
+            "omap_digest": "0xffffffff",
+            "expected_object_size": 0,
+            "expected_write_size": 0,
+            "alloc_hint_flags": 0,
+            "manifest": {
+              "type": 0
+            },
+            "watchers": {},
+            "shard_versions": []
+          }
+        },
+        {
+          "osd": 2,
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x00000000"
+        }
+      ]
     },
     {
       "object": {
@@ -4839,13 +4855,13 @@ EOF
           "pool": 3,
           "namespace": ""
         },
-        "version": "65'8",
+        "version": "78'8",
         "prior_version": "0'0",
-        "last_reqid": "client.4418.0:1",
+        "last_reqid": "client.4470.0:1",
         "user_version": 8,
         "size": 7,
-        "mtime": "2018-04-05 14:32:20.634116",
-        "local_mtime": "2018-04-05 14:32:20.637999",
+        "mtime": "2025-04-28T05:23:39.428058-0500",
+        "local_mtime": "2025-04-28T05:23:39.433754-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4861,7 +4877,8 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
       "shards": [
         {
@@ -4894,22 +4911,22 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x00000000",
           "hashinfo": {
+            "total_chunk_size": 2048,
             "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         }
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
+          }
         }
       ]
     },
@@ -4935,13 +4952,13 @@ EOF
           "pool": 3,
           "namespace": ""
         },
-        "version": "75'10",
-        "prior_version": "75'9",
-        "last_reqid": "client.4482.0:1",
+        "version": "90'10",
+        "prior_version": "90'9",
+        "last_reqid": "client.4542.0:1",
         "user_version": 10,
         "size": 34,
-        "mtime": "2018-04-05 14:32:33.058782",
-        "local_mtime": "2018-04-05 14:32:33.059679",
+        "mtime": "2025-04-28T05:23:55.240961-0500",
+        "local_mtime": "2025-04-28T05:23:55.242367-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -4957,7 +4974,8 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
       "shards": [
         {
@@ -4969,21 +4987,21 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x00000000",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
           }
         },
         {
@@ -4995,21 +5013,21 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x00000000",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
           }
         },
         {
@@ -5021,27 +5039,26 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x00000000",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
           }
         }
       ]
     }
-  ],
-  "epoch": 0
+  ]
 }
 EOF
 
@@ -5049,19 +5066,79 @@ EOF
 
       jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
 {
+  "epoch": 99,
   "inconsistents": [
     {
+      "object": {
+        "name": "EOBJ1",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 1
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "read_error",
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
+      ],
+      "selected_object_info": {
+        "oid": {
+          "oid": "EOBJ1",
+          "key": "",
+          "snapid": -2,
+          "hash": 560836233,
+          "max": 0,
+          "pool": 3,
+          "namespace": ""
+        },
+        "version": "32'1",
+        "prior_version": "0'0",
+        "last_reqid": "client.4210.0:1",
+        "user_version": 1,
+        "size": 7,
+        "mtime": "2025-04-28T02:18:19.605985-0500",
+        "local_mtime": "2025-04-28T02:18:19.607916-0500",
+        "lost": 0,
+        "flags": [
+          "dirty",
+          "data_digest"
+        ],
+        "truncate_seq": 0,
+        "truncate_size": 0,
+        "data_digest": "0x2ddbf8f5",
+        "omap_digest": "0xffffffff",
+        "expected_object_size": 0,
+        "expected_write_size": 0,
+        "alloc_hint_flags": 0,
+        "manifest": {
+          "type": 0
+        },
+        "watchers": {},
+        "shard_versions": []
+      },
       "shards": [
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 9,
           "object_info": {
             "oid": {
               "oid": "EOBJ1",
@@ -5072,13 +5149,13 @@ EOF
               "pool": 3,
               "namespace": ""
             },
-            "version": "27'1",
+            "version": "32'1",
             "prior_version": "0'0",
-            "last_reqid": "client.4192.0:1",
+            "last_reqid": "client.4210.0:1",
             "user_version": 1,
             "size": 7,
-            "mtime": "2018-04-05 14:30:10.688009",
-            "local_mtime": "2018-04-05 14:30:10.691774",
+            "mtime": "2025-04-28T02:18:19.605985-0500",
+            "local_mtime": "2025-04-28T02:18:19.607916-0500",
             "lost": 0,
             "flags": [
               "dirty",
@@ -5094,45 +5171,50 @@ EOF
             "manifest": {
               "type": 0
             },
-            "watchers": {}
-          },
-          "size": 9,
-          "shard": 0,
-          "errors": [
-            "read_error",
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "osd": 1,
-          "primary": true
+            "watchers": {},
+            "shard_versions": []
+          }
         },
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ2",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 2
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "ec_hash_error"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ1",
+          "oid": "EOBJ2",
           "key": "",
           "snapid": -2,
-          "hash": 560836233,
+          "hash": 562812377,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "27'1",
+        "version": "39'2",
         "prior_version": "0'0",
-        "last_reqid": "client.4192.0:1",
-        "user_version": 1,
+        "last_reqid": "client.4248.0:1",
+        "user_version": 2,
         "size": 7,
-        "mtime": "2018-04-05 14:30:10.688009",
-        "local_mtime": "2018-04-05 14:30:10.691774",
+        "mtime": "2025-04-28T02:18:27.810433-0500",
+        "local_mtime": "2025-04-28T02:18:27.815649-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5148,71 +5230,68 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [
-        "read_error",
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
-      ],
-      "object": {
-        "version": 1,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ1"
-      }
-    },
-    {
       "shards": [
         {
-          "size": 2048,
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
           "errors": [
             "ec_hash_error"
           ],
-          "shard": 2,
-          "osd": 0,
-          "primary": false
+          "size": 2048
         },
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 0,
           "osd": 1,
-          "primary": true
+          "primary": true,
+          "shard": 0,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         },
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 1,
           "osd": 2,
-          "primary": false
+          "primary": false,
+          "shard": 1,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ3",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 3
+      },
+      "errors": [],
+      "union_shard_errors": [
+        "missing"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ2",
+          "oid": "EOBJ3",
           "key": "",
           "snapid": -2,
-          "hash": 562812377,
+          "hash": 3125668237,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "33'2",
+        "version": "46'3",
         "prior_version": "0'0",
-        "last_reqid": "client.4224.0:1",
-        "user_version": 2,
+        "last_reqid": "client.4286.0:1",
+        "user_version": 3,
         "size": 7,
-        "mtime": "2018-04-05 14:30:14.152945",
-        "local_mtime": "2018-04-05 14:30:14.154014",
+        "mtime": "2025-04-28T02:18:36.149933-0500",
+        "local_mtime": "2025-04-28T02:18:36.155056-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5223,35 +5302,23 @@ EOF
         "data_digest": "0x2ddbf8f5",
         "omap_digest": "0xffffffff",
         "expected_object_size": 0,
-        "expected_write_size": 0,
-        "alloc_hint_flags": 0,
-        "manifest": {
-          "type": 0
-        },
-        "watchers": {}
-      },
-      "union_shard_errors": [
-        "ec_hash_error"
-      ],
-      "errors": [],
-      "object": {
-        "version": 2,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ2"
-      }
-    },
-    {
+        "expected_write_size": 0,
+        "alloc_hint_flags": 0,
+        "manifest": {
+          "type": 0
+        },
+        "watchers": {},
+        "shard_versions": []
+      },
       "shards": [
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         },
         {
           "osd": 1,
@@ -5262,32 +5329,46 @@ EOF
           ]
         },
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ4",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 6
+      },
+      "errors": [
+        "attr_value_mismatch",
+        "attr_name_mismatch"
       ],
+      "union_shard_errors": [],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ3",
+          "oid": "EOBJ4",
           "key": "",
           "snapid": -2,
-          "hash": 3125668237,
+          "hash": 1618759290,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "39'3",
-        "prior_version": "0'0",
-        "last_reqid": "client.4258.0:1",
-        "user_version": 3,
+        "version": "54'6",
+        "prior_version": "54'5",
+        "last_reqid": "client.4330.0:1",
+        "user_version": 6,
         "size": 7,
-        "mtime": "2018-04-05 14:30:18.875544",
-        "local_mtime": "2018-04-05 14:30:18.880153",
+        "mtime": "2025-04-28T02:18:44.594720-0500",
+        "local_mtime": "2025-04-28T02:18:44.596235-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5303,42 +5384,30 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [
-        "missing"
-      ],
-      "errors": [],
-      "object": {
-        "version": 3,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ3"
-      }
-    },
-    {
       "shards": [
         {
+          "osd": 0,
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "bad-val",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
-          ],
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
-          "osd": 0,
-          "primary": false
+          ]
         },
         {
           "osd": 1,
@@ -5350,14 +5419,14 @@ EOF
           "data_digest": "0x04cfa72f",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key2-EOBJ4",
               "value": "val2-EOBJ4",
-              "name": "key2-EOBJ4"
+              "Base64": false
             }
           ]
         },
@@ -5371,35 +5440,52 @@ EOF
           "data_digest": "0x04cfa72f",
           "attrs": [
             {
-              "Base64": false,
+              "name": "key1-EOBJ4",
               "value": "val1-EOBJ4",
-              "name": "key1-EOBJ4"
+              "Base64": false
             },
             {
-              "Base64": false,
+              "name": "key3-EOBJ4",
               "value": "val3-EOBJ4",
-              "name": "key3-EOBJ4"
+              "Base64": false
             }
           ]
         }
+      ]
+    },
+    {
+      "object": {
+        "name": "EOBJ5",
+        "nspace": "",
+        "locator": "",
+        "snap": "head",
+        "version": 7
+      },
+      "errors": [
+        "size_mismatch"
+      ],
+      "union_shard_errors": [
+        "read_error",
+        "size_mismatch_info",
+        "obj_size_info_mismatch"
       ],
       "selected_object_info": {
         "oid": {
-          "oid": "EOBJ4",
+          "oid": "EOBJ5",
           "key": "",
           "snapid": -2,
-          "hash": 1618759290,
+          "hash": 2918945441,
           "max": 0,
           "pool": 3,
           "namespace": ""
         },
-        "version": "45'6",
-        "prior_version": "45'5",
-        "last_reqid": "client.4296.0:1",
-        "user_version": 6,
+        "version": "71'7",
+        "prior_version": "0'0",
+        "last_reqid": "client.4424.0:1",
+        "user_version": 7,
         "size": 7,
-        "mtime": "2018-04-05 14:30:22.271983",
-        "local_mtime": "2018-04-05 14:30:22.272840",
+        "mtime": "2025-04-28T02:19:03.683994-0500",
+        "local_mtime": "2025-04-28T02:19:03.685505-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5415,33 +5501,29 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
-      "union_shard_errors": [],
-      "errors": [
-        "attr_value_mismatch",
-        "attr_name_mismatch"
-      ],
-      "object": {
-        "version": 6,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ4"
-      }
-    },
-    {
       "shards": [
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
-          "errors": [],
-          "shard": 2,
           "osd": 0,
-          "primary": false
+          "primary": false,
+          "shard": 2,
+          "errors": [],
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         },
         {
+          "osd": 1,
+          "primary": true,
+          "shard": 0,
+          "errors": [
+            "read_error",
+            "size_mismatch_info",
+            "obj_size_info_mismatch"
+          ],
+          "size": 4096,
           "object_info": {
             "oid": {
               "oid": "EOBJ5",
@@ -5452,13 +5534,13 @@ EOF
               "pool": 3,
               "namespace": ""
             },
-            "version": "59'7",
+            "version": "71'7",
             "prior_version": "0'0",
-            "last_reqid": "client.4384.0:1",
+            "last_reqid": "client.4424.0:1",
             "user_version": 7,
             "size": 7,
-            "mtime": "2018-04-05 14:30:35.162395",
-            "local_mtime": "2018-04-05 14:30:35.166390",
+            "mtime": "2025-04-28T02:19:03.683994-0500",
+            "local_mtime": "2025-04-28T02:19:03.685505-0500",
             "lost": 0,
             "flags": [
               "dirty",
@@ -5474,77 +5556,20 @@ EOF
             "manifest": {
               "type": 0
             },
-            "watchers": {}
-          },
-          "size": 4096,
-          "shard": 0,
-          "errors": [
-            "read_error",
-            "size_mismatch_info",
-            "obj_size_info_mismatch"
-          ],
-          "osd": 1,
-          "primary": true
+            "watchers": {},
+            "shard_versions": []
+          }
         },
         {
-          "data_digest": "0x04cfa72f",
-          "omap_digest": "0xffffffff",
-          "size": 2048,
+          "osd": 2,
+          "primary": false,
           "shard": 1,
           "errors": [],
-          "osd": 2,
-          "primary": false
+          "size": 2048,
+          "omap_digest": "0xffffffff",
+          "data_digest": "0x04cfa72f"
         }
-      ],
-      "selected_object_info": {
-        "oid": {
-          "oid": "EOBJ5",
-          "key": "",
-          "snapid": -2,
-          "hash": 2918945441,
-          "max": 0,
-          "pool": 3,
-          "namespace": ""
-        },
-        "version": "59'7",
-        "prior_version": "0'0",
-        "last_reqid": "client.4384.0:1",
-        "user_version": 7,
-        "size": 7,
-        "mtime": "2018-04-05 14:30:35.162395",
-        "local_mtime": "2018-04-05 14:30:35.166390",
-        "lost": 0,
-        "flags": [
-          "dirty",
-          "data_digest"
-        ],
-        "truncate_seq": 0,
-        "truncate_size": 0,
-        "data_digest": "0x2ddbf8f5",
-        "omap_digest": "0xffffffff",
-        "expected_object_size": 0,
-        "expected_write_size": 0,
-        "alloc_hint_flags": 0,
-        "manifest": {
-          "type": 0
-        },
-        "watchers": {}
-      },
-      "union_shard_errors": [
-        "read_error",
-        "size_mismatch_info",
-        "obj_size_info_mismatch"
-      ],
-      "errors": [
-        "size_mismatch"
-      ],
-      "object": {
-        "version": 7,
-        "snap": "head",
-        "locator": "",
-        "nspace": "",
-        "name": "EOBJ5"
-      }
+      ]
     },
     {
       "object": {
@@ -5570,13 +5595,13 @@ EOF
           "pool": 3,
           "namespace": ""
         },
-        "version": "65'8",
+        "version": "78'8",
         "prior_version": "0'0",
-        "last_reqid": "client.4420.0:1",
+        "last_reqid": "client.4462.0:1",
         "user_version": 8,
         "size": 7,
-        "mtime": "2018-04-05 14:30:40.914673",
-        "local_mtime": "2018-04-05 14:30:40.917705",
+        "mtime": "2025-04-28T02:19:11.980798-0500",
+        "local_mtime": "2025-04-28T02:19:11.986446-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5592,7 +5617,8 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
       "shards": [
         {
@@ -5625,21 +5651,21 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x04cfa72f",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
           }
         }
       ]
@@ -5668,13 +5694,13 @@ EOF
           "pool": 3,
           "namespace": ""
         },
-        "version": "75'10",
-        "prior_version": "75'9",
-        "last_reqid": "client.4486.0:1",
+        "version": "90'10",
+        "prior_version": "90'9",
+        "last_reqid": "client.4534.0:1",
         "user_version": 10,
         "size": 34,
-        "mtime": "2018-04-05 14:30:50.995009",
-        "local_mtime": "2018-04-05 14:30:50.996112",
+        "mtime": "2025-04-28T02:19:27.775012-0500",
+        "local_mtime": "2025-04-28T02:19:27.776394-0500",
         "lost": 0,
         "flags": [
           "dirty",
@@ -5690,7 +5716,8 @@ EOF
         "manifest": {
           "type": 0
         },
-        "watchers": {}
+        "watchers": {},
+        "shard_versions": []
       },
       "shards": [
         {
@@ -5702,22 +5729,22 @@ EOF
           ],
           "size": 2048,
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 80717615,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 80717615,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
-         }
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 80717615
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 80717615
+              }
+            ]
+          }
         },
         {
           "osd": 1,
@@ -5728,21 +5755,21 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x5b7455a8",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
           }
         },
         {
@@ -5754,27 +5781,26 @@ EOF
           "omap_digest": "0xffffffff",
           "data_digest": "0x5b7455a8",
           "hashinfo": {
-           "cumulative_shard_hashes": [
-            {
-              "hash": 1534350760,
-              "shard": 0
-            },
-            {
-              "hash": 1534491824,
-              "shard": 1
-            },
-            {
-              "hash": 1534350760,
-              "shard": 2
-            }
-           ],
-           "total_chunk_size": 2048
+            "total_chunk_size": 2048,
+            "cumulative_shard_hashes": [
+              {
+                "shard": 0,
+                "hash": 1534350760
+              },
+              {
+                "shard": 1,
+                "hash": 1534491824
+              },
+              {
+                "shard": 2,
+                "hash": 1534350760
+              }
+            ]
           }
         }
       ]
     }
-  ],
-  "epoch": 0
+  ]
 }
 EOF
 
-- 
2.39.5