From: Jason Dillaman Date: Thu, 3 Oct 2019 18:30:46 +0000 (-0400) Subject: qa/workunits/rbd: tweak mirror test case to handle peer status X-Git-Tag: v15.1.0~1245^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=eaec1786eeb8eefeb9c1dbd64d3f381cc9626345;p=ceph.git qa/workunits/rbd: tweak mirror test case to handle peer status Signed-off-by: Jason Dillaman --- diff --git a/qa/suites/rbd/mirror-thrash/clients/.qa b/qa/suites/rbd/mirror-thrash/clients/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/clients/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/mirror-thrash/clients/mirror.yaml b/qa/suites/rbd/mirror-thrash/clients/mirror.yaml new file mode 100644 index 00000000000..1b6808d8572 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/clients/mirror.yaml @@ -0,0 +1,36 @@ +meta: +- desc: configure the permissions for client.mirror +overrides: + ceph: + conf: + client: + rbd default features: 125 + debug rbd: 20 + debug rbd_mirror: 15 + log to stderr: false + # override to make these names predictable + client.mirror.0: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.1: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.2: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.3: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid +tasks: +- exec: + cluster1.client.mirror.0: + - "sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'" diff --git a/qa/suites/rbd/mirror-thrash/users/.qa b/qa/suites/rbd/mirror-thrash/users/.qa deleted file mode 120000 index a602a0353e7..00000000000 --- a/qa/suites/rbd/mirror-thrash/users/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/mirror-thrash/users/mirror.yaml b/qa/suites/rbd/mirror-thrash/users/mirror.yaml deleted file mode 100644 index 8de3eb2b8f3..00000000000 --- a/qa/suites/rbd/mirror-thrash/users/mirror.yaml +++ /dev/null @@ -1,23 +0,0 @@ -meta: -- desc: configure the permissions for client.mirror -overrides: - ceph: - conf: - client: - rbd default features: 125 - debug rbd: 20 - debug rbd_mirror: 15 - log to stderr: false - # override to make these names predictable - client.mirror.0: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid - client.mirror.1: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid - client.mirror.2: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid - client.mirror.3: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid diff --git a/qa/suites/rbd/mirror/clients/+ b/qa/suites/rbd/mirror/clients/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rbd/mirror/clients/mirror-extra.yaml b/qa/suites/rbd/mirror/clients/mirror-extra.yaml new file mode 100644 index 00000000000..5eed5cfcd97 --- /dev/null +++ b/qa/suites/rbd/mirror/clients/mirror-extra.yaml @@ -0,0 +1,24 @@ +meta: +- desc: configure the permissions for client.mirror +overrides: + ceph: + conf: + # override to make these names predictable + client.mirror.4: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.5: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.6: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid +tasks: +- exec: + cluster1.client.mirror.0: + - "sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'" + - "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'" diff --git a/qa/suites/rbd/mirror/clients/mirror.yaml b/qa/suites/rbd/mirror/clients/mirror.yaml new file mode 120000 index 00000000000..0dfb0c78543 --- /dev/null +++ b/qa/suites/rbd/mirror/clients/mirror.yaml @@ -0,0 +1 @@ +../../mirror-thrash/clients/mirror.yaml \ No newline at end of file diff --git a/qa/suites/rbd/mirror/users b/qa/suites/rbd/mirror/users deleted file mode 120000 index 8d9d0d2c336..00000000000 --- a/qa/suites/rbd/mirror/users +++ /dev/null @@ -1 +0,0 @@ -../mirror-thrash/users \ No newline at end of file diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml index 7aa8d548c91..7347f7f7602 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml @@ -5,16 +5,6 @@ overrides: conf: client: rbd mirror image policy type: none - # override to make these names predictable - client.mirror.4: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid - client.mirror.5: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid - client.mirror.6: - admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok - pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid tasks: - workunit: clients: diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index a76ad5860a8..8f979f581fb 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -360,38 +360,38 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} testlog "TEST: non-default namespace image mirroring" testlog " - replay" -create_image ${CLUSTER2} ${POOL}/ns1 ${image} -create_image ${CLUSTER2} ${POOL}/ns2 ${image} -enable_mirror ${CLUSTER2} ${POOL}/ns2 ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL}/ns1 ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL}/ns2 ${image} -write_image ${CLUSTER2} ${POOL}/ns1 ${image} 100 -write_image ${CLUSTER2} ${POOL}/ns2 ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/ns1 ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/ns2 ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/ns1 ${image} 'up+replaying' 'master_position' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/ns2 ${image} 'up+replaying' 'master_position' -compare_images ${POOL}/ns1 ${image} -compare_images ${POOL}/ns2 ${image} +create_image ${CLUSTER2} ${POOL}/${NS1} ${image} +create_image ${CLUSTER2} ${POOL}/${NS2} ${image} +enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 +write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'master_position' +compare_images ${POOL}/${NS1} ${image} +compare_images ${POOL}/${NS2} ${image} testlog " - disable mirroring / delete image" -remove_image_retry ${CLUSTER2} ${POOL}/ns1 ${image} -disable_mirror ${CLUSTER2} ${POOL}/ns2 ${image} -wait_for_image_present ${CLUSTER1} ${POOL}/ns1 ${image} 'deleted' -wait_for_image_present ${CLUSTER1} ${POOL}/ns2 ${image} 'deleted' +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} +disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' +wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' testlog " - data pool" dp_image=test_data_pool -create_image ${CLUSTER2} ${POOL}/ns1 ${dp_image} 128 --data-pool ${PARENT_POOL} -data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/ns1 ${dp_image}) +create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" -wait_for_image_replay_started ${CLUSTER1} ${POOL}/ns1 ${dp_image} -data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/ns1 ${dp_image}) +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" -write_image ${CLUSTER2} ${POOL}/ns1 ${dp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/ns1 ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/ns1 ${dp_image} 'up+replaying' 'master_position' -compare_images ${POOL}/ns1 ${dp_image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'master_position' +compare_images ${POOL}/${NS1} ${dp_image} testlog "TEST: simple image resync" request_resync_image ${CLUSTER1} ${POOL} ${image} image_id diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 027c8af91ed..c364dd7ce6b 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -82,6 +82,8 @@ CLUSTER2=cluster2 PEER_CLUSTER_SUFFIX= POOL=mirror PARENT_POOL=mirror_parent +NS1=ns1 +NS2=ns2 TEMPDIR= CEPH_ID=${CEPH_ID:-mirror} MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} @@ -206,20 +208,6 @@ create_users() done } -update_users() -{ - local cluster=$1 - - CEPH_ARGS='' ceph --cluster "${cluster}" \ - auth caps client.${CEPH_ID} \ - mon 'profile rbd' osd 'profile rbd' - for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do - CEPH_ARGS='' ceph --cluster "${cluster}" \ - auth caps client.${MIRROR_USER_ID_PREFIX}${instance} \ - mon 'profile rbd-mirror' osd 'profile rbd' - done -} - setup_cluster() { local cluster=$1 @@ -259,14 +247,18 @@ setup_pools() CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL} CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL} - rbd --cluster ${cluster} mirror pool enable ${POOL} pool + if [ -n "${RBD_MIRROR_CONFIG_KEY}" ]; then + PEER_CLUSTER_SUFFIX=-DNE + fi + + rbd --cluster ${cluster} mirror pool enable --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} pool rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image - rbd --cluster ${cluster} namespace create ${POOL}/ns1 - rbd --cluster ${cluster} namespace create ${POOL}/ns2 + rbd --cluster ${cluster} namespace create ${POOL}/${NS1} + rbd --cluster ${cluster} namespace create ${POOL}/${NS2} - rbd --cluster ${cluster} mirror pool enable ${POOL}/ns1 pool - rbd --cluster ${cluster} mirror pool enable ${POOL}/ns2 image + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} pool + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then @@ -281,14 +273,14 @@ setup_pools() admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file} - rbd --cluster ${cluster} mirror pool peer add ${POOL} client.${CEPH_ID}@${remote_cluster}-DNE \ + rbd --cluster ${cluster} mirror pool peer add ${POOL} \ + client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} \ --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file} - uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}-DNE) + uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} \ + client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX}) rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr} rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file} - - PEER_CLUSTER_SUFFIX=-DNE fi fi } @@ -314,9 +306,6 @@ setup() if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then setup_cluster "${CLUSTER1}" setup_cluster "${CLUSTER2}" - else - update_users "${CLUSTER1}" - update_users "${CLUSTER2}" fi setup_pools "${CLUSTER1}" "${CLUSTER2}" @@ -477,7 +466,7 @@ all_admin_daemons() status() { - local cluster daemon image_pool image + local cluster daemon image_pool image_ns image for cluster in ${CLUSTER1} ${CLUSTER2} do @@ -489,23 +478,34 @@ status() for image_pool in ${POOL} ${PARENT_POOL} do - echo "${cluster} ${image_pool} images" - rbd --cluster ${cluster} -p ${image_pool} ls -l - echo - - echo "${cluster} ${image_pool} mirror pool status" - rbd --cluster ${cluster} -p ${image_pool} mirror pool status --verbose - echo + for image_ns in "" "/${NS1}" "/${NS2}" + do + echo "${cluster} ${image_pool}${image_ns} images" + rbd --cluster ${cluster} -p ${image_pool}{$image_ns} ls -l + echo - for image in `rbd --cluster ${cluster} -p ${image_pool} ls 2>/dev/null` - do - echo "image ${image} info" - rbd --cluster ${cluster} -p ${image_pool} info ${image} + echo "${cluster} ${image_pool}${image_ns} mirror pool info" + rbd --cluster ${cluster} -p ${image_pool}${image_ns} mirror pool info echo - echo "image ${image} journal status" - rbd --cluster ${cluster} -p ${image_pool} journal status --image ${image} + + echo "${cluster} ${image_pool}${image_ns} mirror pool status" + rbd --cluster ${cluster} -p ${image_pool}${image_ns} mirror pool status --verbose echo - done + + for image in `rbd --cluster ${cluster} -p ${image_pool}${image_ns} ls 2>/dev/null` + do + echo "image ${image} info" + rbd --cluster ${cluster} -p ${image_pool}${image_ns} info ${image} + echo + echo "image ${image} journal status" + rbd --cluster ${cluster} -p ${image_pool}${image_ns} journal status --image ${image} + echo + done + + echo "${cluster} ${image_pool}${image_ns} rbd_mirroring omap vals" + rados --cluster ${cluster} -p ${image_pool}${image_ns} listomapvals rbd_mirroring + echo + done done done @@ -703,8 +703,8 @@ test_status_in_pool_dir() local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status) rbd --cluster ${cluster} mirror image status ${pool}/${image} | tee ${status_log} >&2 - grep "state: .*${state_pattern}" ${status_log} || return 1 - grep "description: .*${description_pattern}" ${status_log} || return 1 + grep "^ state: .*${state_pattern}" ${status_log} || return 1 + grep "^ description: .*${description_pattern}" ${status_log} || return 1 if [ -n "${service_pattern}" ]; then grep "service: *${service_pattern}" ${status_log} || return 1 @@ -716,7 +716,7 @@ test_status_in_pool_dir() # recheck using `mirror pool status` command to stress test it. - local last_update="$(sed -nEe 's/^ *last_update: *(.*) *$/\1/p' ${status_log})" + local last_update="$(sed -nEe 's/^ last_update: *(.*) *$/\1/p' ${status_log})" test_mirror_pool_status_verbose \ ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" && return 0