osd: retry to read object attrs at EC recovery

In EC recovery read, if the object's attrs read failed or with errors, we erase the attrs we have read and
try to read it again from left shards. This will make the primary osd get the object's attrs correct and
avoid assert.

Signed-off-by: xiaofei cui <cuixiaofei@sangfor.com>
(cherry picked from commit 3eb1679b1fadb6adb6ff4ed3f1f6069a85f4bbcd)

diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh
index 32bef54ef45fd6dae563cfc4c4fc9ddc14abc264..ce037aacafef1f085a978609a0d071489f486cd1 100755 (executable)
--- a/qa/standalone/erasure-code/test-erasure-eio.sh
+++ b/qa/standalone/erasure-code/test-erasure-eio.sh
@@ -353,6 +353,37 @@ function TEST_rados_get_with_subreadall_eio_shard_1() {
     delete_erasure_coded_pool $poolname
 }
 
+# Test recovery the object attr read error
+function TEST_ec_object_attr_read_error() {
+    local dir=$1
+    local objname=myobject
+
+    setup_osds 7 || return 1
+
+    local poolname=pool-jerasure
+    create_erasure_coded_pool $poolname 3 2 || return 1
+
+    local primary_osd=$(get_primary $poolname $objname)
+    # Kill primary OSD
+    kill_daemons $dir TERM osd.${primary_osd} >&2 < /dev/null || return 1
+
+    # Write data
+    rados_put $dir $poolname $objname || return 1
+
+    # Inject eio, shard 1 is the one read attr
+    inject_eio ec mdata $poolname $objname $dir 1 || return 1
+
+    # Restart OSD
+    run_osd $dir ${primary_osd} || return 1
+
+    # Cluster should recover this object
+    wait_for_clean || return 1
+
+    rados_get $dir $poolname myobject || return 1
+
+    delete_erasure_coded_pool $poolname
+}
+
 # Test recovery the first k copies aren't all available
 function TEST_ec_single_recovery_error() {
     local dir=$1

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 23e5a50f0166ceb891f11e602ea81503f0412307..a5b8e7461bece73eba05080220453f35d670e3e1 100644 (file)
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1077,6 +1077,8 @@ error:
        *i, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
       reply->attrs_read[*i]);
     if (r < 0) {
+      // If we read error, we should not return the attrs too.
+      reply->attrs_read.erase(*i);
       reply->buffers_read.erase(*i);
       reply->errors[*i] = r;
     }
@@ -2371,13 +2373,21 @@ int ECBackend::send_all_remaining_reads(
   GenContext<pair<RecoveryMessages *, read_result_t& > &> *c =
     rop.to_read.find(hoid)->second.cb;
 
+  // (Note cuixf) If we need to read attrs and we read failed, try to read again.
+  bool want_attrs =
+    rop.to_read.find(hoid)->second.want_attrs &&
+    (!rop.complete[hoid].attrs || rop.complete[hoid].attrs->empty());
+  if (want_attrs) {
+    dout(10) << __func__ << " want attrs again" << dendl;
+  }
+
   rop.to_read.erase(hoid);
   rop.to_read.insert(make_pair(
       hoid,
       read_request_t(
        offsets,
        shards,
-       false,
+       want_attrs,
        c)));
   do_read_op(rop);
   return 0;