qa/workunits/rbd: increase writes in large image count test

This will help to test edge cases where the remote image does
or does not own the exclusive lock when the sync starts.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit d16698f96c65b9627cc1f64ef80b7f1e39d69d45)

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 6a24d0ec5ab2860faad2122a7d702df2bd2263b9..9478cc8f9d1c209edd71c9d4f1e035451ae37bd8 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -408,14 +408,28 @@ wait_for_replay_complete()
     local cluster=$2
     local pool=$3
     local image=$4
-    local s master_pos mirror_pos
-
-    for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
-       sleep ${s}
-       flush "${local_cluster}" "${pool}" "${image}"
-       master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
-       mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
-       test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+    local s master_pos mirror_pos last_mirror_pos
+    local master_tag master_entry mirror_tag mirror_entry
+
+    while true; do
+        for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
+           sleep ${s}
+           flush "${local_cluster}" "${pool}" "${image}"
+           master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
+           mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
+           test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+            test "${mirror_pos}" != "${last_mirror_pos}" && break
+        done
+
+        test "${mirror_pos}" = "${last_mirror_pos}" && return 1
+        last_mirror_pos="${mirror_pos}"
+
+        # handle the case where the mirror is ahead of the master
+        master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0
     done
     return 1
 }

diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
index a0e34913066b860beac2c05e259ca565e5e811fd..1caea2332607494551d2a65dc84b96bea0bfe245 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_stress.sh
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -2,6 +2,11 @@
 #
 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
 #
+# The following additional environment variables affect the test:
+#
+#  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
+#                             tool during the many image test
+#
 
 IMAGE_COUNT=50
 export LOCKDEP=0
@@ -40,23 +45,43 @@ compare_image_snaps()
     rm -f ${rmt_export} ${loc_export}
 }
 
-wait_for_pool_healthy()
+wait_for_pool_images()
 {
     local cluster=$1
     local pool=$2
     local image_count=$3
     local s
     local count
+    local last_count=0
+
+    while true; do
+        for s in `seq 1 40`; do
+            count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
+            test "${count}" = "${image_count}" && return 0
+
+            # reset timeout if making forward progress
+            test $count -gt $last_count && break
+            sleep 30
+        done
+
+        test $count -eq $last_count && return 1
+        $last_count=$count
+    done
+    return 1
+}
+
+wait_for_pool_healthy()
+{
+    local cluster=$1
+    local pool=$2
+    local s
     local state
 
     for s in `seq 1 40`; do
+        state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
+        test "${state}" = "ERROR" && return 1
+        test "${state}" = "OK" && return 0
        sleep 30
-        count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ')
-        test "${count}" = "images: ${image_count} total" || continue
-
-        state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:')
-        test "${state}" = "health: ERROR" && return 1
-        test "${state}" = "health: OK" && return 0
     done
     return 1
 }
@@ -91,29 +116,40 @@ remove_image ${CLUSTER2} ${POOL} ${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 
 testlog "TEST: create many images"
+snap_name="snap"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
   create_image ${CLUSTER2} ${POOL} ${image} '128M'
-  write_image ${CLUSTER2} ${POOL} ${image} 100
+  if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
+    write_image ${CLUSTER2} ${POOL} ${image} 100
+  else
+    stress_write_image ${CLUSTER2} ${POOL} ${image}
+  fi
 done
 
-wait_for_pool_healthy ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
-wait_for_pool_healthy ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER2} ${POOL}
+
+wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER1} ${POOL}
 
 testlog "TEST: compare many images"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
+  create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-  compare_images ${POOL} ${image}
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
+  compare_image_snaps ${POOL} ${image} ${snap_name}
 done
 
 testlog "TEST: delete many images"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
+  remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
   remove_image ${CLUSTER2} ${POOL} ${image}
 done