tests: verify erasure code read / write after remapping

Implement the following scenario:

 * create an erasure coded pool and write an object

 * mark two osd used to store the object out
 * wait until the PG is clean
 * read the object and compare it with the original

 * mark the two osd back in
 * wait until the PG is clean
 * read the object and compare it with the original

The scenario is run with the jerasure plugin only because the code path
involved does not vary depending on the erasure code plugin.

http://tracker.ceph.com/issues/11667 Fixes: #11667

Signed-off-by: Loic Dachary <ldachary@redhat.com>

diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh
index ed699b024e3d07973220a4769de72271ab541c4c..cbc25bf32558e259c47195e37197e9a99b0d83ca 100755 (executable)
--- a/src/test/erasure-code/test-erasure-code.sh
+++ b/src/test/erasure-code/test-erasure-code.sh
@@ -86,7 +86,7 @@ function rados_put_get() {
     rm $dir/COPY
 
     #
-    # take out the first OSD used to store the object and
+    # take out an OSD used to store the object and
     # check the object can still be retrieved, which implies
     # recovery
     #
@@ -101,6 +101,57 @@ function rados_put_get() {
     rm $dir/ORIGINAL
 }
 
+function rados_osds_out_in() {
+    local dir=$1
+    local poolname=$2
+    local objname=${3:-SOMETHING}
+
+
+    for marker in FFFF GGGG HHHH IIII ; do
+        printf "%*s" 1024 $marker
+    done > $dir/ORIGINAL
+
+    #
+    # get and put an object, compare they are equal
+    #
+    ./rados --pool $poolname put $objname $dir/ORIGINAL || return 1
+    ./rados --pool $poolname get $objname $dir/COPY || return 1
+    diff $dir/ORIGINAL $dir/COPY || return 1
+    rm $dir/COPY
+
+    #
+    # take out two OSDs used to store the object, wait for the cluster
+    # to be clean (i.e. all PG are clean and active) again which
+    # implies the PG have been moved to use the remaining OSDs.  Check
+    # the object can still be retrieved.
+    #
+    local osds_list=$(get_osds $poolname $objname)
+    local -a osds=($osds_list)
+    for osd in 0 1 ; do
+        ./ceph osd out ${osds[$osd]} || return 1
+    done
+    wait_for_clean || return 1
+    #
+    # verify the object is no longer mapped to the osds that are out
+    #
+    for osd in 0 1 ; do
+        ! get_osds $poolname $objname | grep '\<'${osds[$osd]}'\>' || return 1
+    done
+    ./rados --pool $poolname get $objname $dir/COPY || return 1
+    diff $dir/ORIGINAL $dir/COPY || return 1
+    #
+    # bring the osds back in, , wait for the cluster
+    # to be clean (i.e. all PG are clean and active) again which
+    # implies the PG go back to using the same osds as before
+    #
+    for osd in 0 1 ; do
+        ./ceph osd in ${osds[$osd]} || return 1
+    done
+    wait_for_clean || return 1
+    test "$osds_list" = "$(get_osds $poolname $objname)" || return 1
+    rm $dir/ORIGINAL
+}
+
 function TEST_rados_put_get_lrc_advanced() {
     local dir=$1
     local poolname=pool-lrc-a
@@ -173,6 +224,7 @@ function TEST_rados_put_get_jerasure() {
         || return 1
 
     rados_put_get $dir $poolname || return 1
+    rados_osds_out_in $dir $poolname || return 1
 
     delete_pool $poolname
     ./ceph osd erasure-code-profile rm $profile