From: Jason Dillaman <dillaman@redhat.com>
Date: Wed, 15 Jun 2016 21:49:54 +0000 (-0400)
Subject: qa/workunits/rbd: increase writes in large image count test
X-Git-Tag: v11.0.0~140^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d16698f96c65b9627cc1f64ef80b7f1e39d69d45;p=ceph.git

qa/workunits/rbd: increase writes in large image count test

This will help to test edge cases where the remote image does
or does not own the exclusive lock when the sync starts.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 8094bd8de42..3e425fac123 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -408,14 +408,28 @@ wait_for_replay_complete()
     local cluster=$2
     local pool=$3
     local image=$4
-    local s master_pos mirror_pos
-
-    for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
-	sleep ${s}
-	flush "${local_cluster}" "${pool}" "${image}"
-	master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
-	mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
-	test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+    local s master_pos mirror_pos last_mirror_pos
+    local master_tag master_entry mirror_tag mirror_entry
+
+    while true; do
+        for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
+	    sleep ${s}
+	    flush "${local_cluster}" "${pool}" "${image}"
+	    master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
+	    mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
+	    test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+            test "${mirror_pos}" != "${last_mirror_pos}" && break
+        done
+
+        test "${mirror_pos}" = "${last_mirror_pos}" && return 1
+        last_mirror_pos="${mirror_pos}"
+
+        # handle the case where the mirror is ahead of the master
+        master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0
     done
     return 1
 }
diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
index a8561eed9d8..a2565d4886a 100755
--- a/qa/workunits/rbd/rbd_mirror_stress.sh
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -2,6 +2,11 @@
 #
 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
 #
+# The following additional environment variables affect the test:
+#
+#  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
+#                             tool during the many image test
+#
 
 IMAGE_COUNT=50
 export LOCKDEP=0
@@ -39,23 +44,43 @@ compare_image_snaps()
     cmp ${rmt_export} ${loc_export}
 }
 
-wait_for_pool_healthy()
+wait_for_pool_images()
 {
     local cluster=$1
     local pool=$2
     local image_count=$3
     local s
     local count
+    local last_count=0
+
+    while true; do
+        for s in `seq 1 40`; do
+            count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
+            test "${count}" = "${image_count}" && return 0
+
+            # reset timeout if making forward progress
+            test $count -gt $last_count && break
+            sleep 30
+        done
+
+        test $count -eq $last_count && return 1
+        $last_count=$count
+    done
+    return 1
+}
+
+wait_for_pool_healthy()
+{
+    local cluster=$1
+    local pool=$2
+    local s
     local state
 
     for s in `seq 1 40`; do
+        state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
+        test "${state}" = "ERROR" && return 1
+        test "${state}" = "OK" && return 0
 	sleep 30
-        count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ')
-        test "${count}" = "images: ${image_count} total" || continue
-
-        state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:')
-        test "${state}" = "health: ERROR" && return 1
-        test "${state}" = "health: OK" && return 0
     done
     return 1
 }
@@ -90,29 +115,40 @@ remove_image ${CLUSTER2} ${POOL} ${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 
 testlog "TEST: create many images"
+snap_name="snap"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
   create_image ${CLUSTER2} ${POOL} ${image} '128M'
-  write_image ${CLUSTER2} ${POOL} ${image} 100
+  if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
+    write_image ${CLUSTER2} ${POOL} ${image} 100
+  else
+    stress_write_image ${CLUSTER2} ${POOL} ${image}
+  fi
 done
 
-wait_for_pool_healthy ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
-wait_for_pool_healthy ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER2} ${POOL}
+
+wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER1} ${POOL}
 
 testlog "TEST: compare many images"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
+  create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-  compare_images ${POOL} ${image}
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
+  compare_image_snaps ${POOL} ${image} ${snap_name}
 done
 
 testlog "TEST: delete many images"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
+  remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
   remove_image ${CLUSTER2} ${POOL} ${image}
 done