test/rgw: wait for metadata sync

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/test/rgw/test-rgw-common.sh b/src/test/rgw/test-rgw-common.sh
index 9a50fc146e8a38c0fdda999081dfebb7e4131ae0..6193822eb94c885a5d046549b3ce52ba0cfae789 100644 (file)
--- a/src/test/rgw/test-rgw-common.sh
+++ b/src/test/rgw/test-rgw-common.sh
@@ -5,7 +5,8 @@ function var_to_python_json_index {
 }
 
 function json_extract {
-var=$(var_to_python_json_index $1)
+var=""
+[ "$1" != "" ] && var=$(var_to_python_json_index $1)
 shift
 python - <<END
 import json
@@ -29,7 +30,7 @@ python - <<END
 arr=$@
 s='( '
 for x in arr:
-    s += str(x$var) + ' '
+    s += '"' + str(x$var) + '" '
 s += ')'
 print s
 END

diff --git a/src/test/rgw/test-rgw-meta-sync.sh b/src/test/rgw/test-rgw-meta-sync.sh
index c2062ddc587422a339d15e7d553485e17eb2cab7..3882dcb10d4f4dcf26b2544b5f91adec71779971 100755 (executable)
--- a/src/test/rgw/test-rgw-meta-sync.sh
+++ b/src/test/rgw/test-rgw-meta-sync.sh
@@ -4,20 +4,59 @@
 
 set -e
 
-meta_sync_status_json=`$(rgw_admin 2) metadata sync status`
+function get_metadata_sync_status {
+  id=$1
 
+  meta_sync_status_json=`$(rgw_admin $id) metadata sync status`
 
-global_sync_status=$(json_extract sync_status.info.status $meta_sync_status_json)
-num_shards=$(json_extract sync_status.info.num_shards $meta_sync_status_json)
+  global_sync_status=$(json_extract sync_status.info.status $meta_sync_status_json)
+  num_shards=$(json_extract sync_status.info.num_shards $meta_sync_status_json)
 
-echo "sync_status: $global_sync_status"
+  echo "sync_status: $global_sync_status"
 
-sync_markers=$(json_extract sync_status.markers $meta_sync_status_json)
+  sync_markers=$(json_extract sync_status.markers $meta_sync_status_json)
 
-# num_shards=$(python_array_len $sync_markers)
+  # num_shards=$(python_array_len $sync_markers)
 
-# echo $num_shards
+  # echo $num_shards
 
-sync_states=$(project_python_array_field val.state $sync_markers)
+  sync_states=$(project_python_array_field val.state $sync_markers)
+  eval secondary_status=$(project_python_array_field val.marker $sync_markers)
+}
+
+function get_metadata_log_status {
+  master_id=$1
+
+  master_mdlog_status_json=`$(rgw_admin $master_id) mdlog status`
+  master_meta_status=$(json_extract "" $master_mdlog_status_json)
+
+  eval master_status=$(project_python_array_field marker $master_meta_status)
+}
+
+function wait_for_meta_sync {
+  master_id=$1
+  id=$2
+
+  get_metadata_log_status $master_id
+  echo "master_status=${master_status[*]}"
+
+  while true; do
+    get_metadata_sync_status $id
+
+    echo "secondary_status=${secondary_status[*]}"
+
+    fail=0
+    for i in `seq 0 $((num_shards-1))`; do
+      if [ "${master_status[$i]}" \> "${secondary_status[$i]}" ]; then
+        echo "shard $i not done syncing (${master_status[$i]} > ${secondary_status[$i]})"
+        fail=1
+        break
+      fi
+    done
+
+    [ $fail -eq 0 ] && echo "Success" && return || echo "Sync not complete"
+
+    sleep 5
+  done
+}
 
-# echo ${sync_states[*]}

diff --git a/src/test/rgw/test-rgw-multisite.sh b/src/test/rgw/test-rgw-multisite.sh
index 921d43376af9355e89ecfaf179e097d3978dd674..2022f85248195c9fa1e4f042ba85f7549334038b 100755 (executable)
--- a/src/test/rgw/test-rgw-multisite.sh
+++ b/src/test/rgw/test-rgw-multisite.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . "`dirname $0`/test-rgw-common.sh"
+. "`dirname $0`/test-rgw-meta-sync.sh"
 
 set -e
 
@@ -33,3 +34,5 @@ x $(start_ceph_cluster 2) -n
 # create new zone, start rgw
 init_zone_in_existing_zg 2 $realm_name $zg $zone1 $zone1_port $system_access_key $system_secret
 
+
+wait_for_meta_sync 1 2