From 2311eff6225a47634ef48924034b9a2a8249d74c Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 18 Apr 2018 15:14:24 -0400 Subject: [PATCH] qa/suites/rbd: thrash rbd-mirror daemon during tests Fixes: http://tracker.ceph.com/issues/18753 Signed-off-by: Jason Dillaman --- qa/suites/rbd/mirror-ha/base | 1 - qa/suites/rbd/mirror-ha/cluster | 1 - qa/suites/rbd/mirror-ha/msgr-failures | 1 - qa/suites/rbd/mirror-ha/objectstore | 1 - .../workloads/rbd-mirror-ha-workunit.yaml | 16 - qa/suites/rbd/{mirror-ha => mirror-thrash}/% | 0 .../base/install.yaml | 0 .../rbd/{mirror => mirror-thrash}/cluster/+ | 0 .../rbd/mirror-thrash/cluster/2-node.yaml | 25 ++ .../cluster/openstack.yaml | 0 qa/suites/rbd/mirror-thrash/msgr-failures | 1 + qa/suites/rbd/mirror-thrash/objectstore | 1 + .../rbd-mirror/four-per-cluster.yaml | 23 ++ qa/suites/rbd/mirror-thrash/users/mirror.yaml | 18 ++ .../workloads/rbd-mirror-stress-workunit.yaml | 1 + .../workloads/rbd-mirror-workunit.yaml | 12 + qa/suites/rbd/mirror/base | 1 + qa/suites/rbd/mirror/cluster/2-node.yaml | 16 +- qa/suites/rbd/mirror/msgr-failures | 2 +- qa/suites/rbd/mirror/objectstore | 2 +- .../mirror/rbd-mirror/one-per-cluster.yaml | 19 -- qa/suites/rbd/mirror/users | 1 + .../workloads/rbd-mirror-ha-workunit.yaml | 24 ++ .../mirror/workloads/rbd-mirror-workunit.yaml | 2 +- qa/workunits/rbd/rbd_mirror.sh | 177 +++++------ qa/workunits/rbd/rbd_mirror_ha.sh | 4 +- qa/workunits/rbd/rbd_mirror_helpers.sh | 281 +++++++++++------- qa/workunits/rbd/rbd_mirror_stress.sh | 4 +- 28 files changed, 401 insertions(+), 233 deletions(-) delete mode 120000 qa/suites/rbd/mirror-ha/base delete mode 120000 qa/suites/rbd/mirror-ha/cluster delete mode 120000 qa/suites/rbd/mirror-ha/msgr-failures delete mode 120000 qa/suites/rbd/mirror-ha/objectstore delete mode 100644 qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml rename qa/suites/rbd/{mirror-ha => mirror-thrash}/% (100%) rename qa/suites/rbd/{mirror => mirror-thrash}/base/install.yaml (100%) rename qa/suites/rbd/{mirror => mirror-thrash}/cluster/+ (100%) create mode 100644 qa/suites/rbd/mirror-thrash/cluster/2-node.yaml rename qa/suites/rbd/{mirror => mirror-thrash}/cluster/openstack.yaml (100%) create mode 120000 qa/suites/rbd/mirror-thrash/msgr-failures create mode 120000 qa/suites/rbd/mirror-thrash/objectstore create mode 100644 qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml create mode 100644 qa/suites/rbd/mirror-thrash/users/mirror.yaml rename qa/suites/rbd/{mirror => mirror-thrash}/workloads/rbd-mirror-stress-workunit.yaml (91%) create mode 100644 qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml create mode 120000 qa/suites/rbd/mirror/base delete mode 100644 qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml create mode 120000 qa/suites/rbd/mirror/users create mode 100644 qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml diff --git a/qa/suites/rbd/mirror-ha/base b/qa/suites/rbd/mirror-ha/base deleted file mode 120000 index 09e88d419b9aa..0000000000000 --- a/qa/suites/rbd/mirror-ha/base +++ /dev/null @@ -1 +0,0 @@ -../mirror/base \ No newline at end of file diff --git a/qa/suites/rbd/mirror-ha/cluster b/qa/suites/rbd/mirror-ha/cluster deleted file mode 120000 index 47e95a264fd79..0000000000000 --- a/qa/suites/rbd/mirror-ha/cluster +++ /dev/null @@ -1 +0,0 @@ -../mirror/cluster \ No newline at end of file diff --git a/qa/suites/rbd/mirror-ha/msgr-failures b/qa/suites/rbd/mirror-ha/msgr-failures deleted file mode 120000 index c0ae027eacaeb..0000000000000 --- a/qa/suites/rbd/mirror-ha/msgr-failures +++ /dev/null @@ -1 +0,0 @@ -../mirror/msgr-failures \ No newline at end of file diff --git a/qa/suites/rbd/mirror-ha/objectstore b/qa/suites/rbd/mirror-ha/objectstore deleted file mode 120000 index 1a5e90ea59be5..0000000000000 --- a/qa/suites/rbd/mirror-ha/objectstore +++ /dev/null @@ -1 +0,0 @@ -../mirror/objectstore \ No newline at end of file diff --git a/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml b/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml deleted file mode 100644 index 406318f81c205..0000000000000 --- a/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml +++ /dev/null @@ -1,16 +0,0 @@ -meta: -- desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon -tasks: -- exec: - cluster1.client.mirror: - - ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd' osd 'profile rbd' - cluster2.client.mirror: - - ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd' osd 'profile rbd' -- workunit: - clients: - cluster1.client.mirror: [rbd/rbd_mirror_ha.sh] - env: - # override workunit setting of CEPH_ARGS='--cluster' - CEPH_ARGS: '' - RBD_MIRROR_USE_EXISTING_CLUSTER: '1' - timeout: 6h diff --git a/qa/suites/rbd/mirror-ha/% b/qa/suites/rbd/mirror-thrash/% similarity index 100% rename from qa/suites/rbd/mirror-ha/% rename to qa/suites/rbd/mirror-thrash/% diff --git a/qa/suites/rbd/mirror/base/install.yaml b/qa/suites/rbd/mirror-thrash/base/install.yaml similarity index 100% rename from qa/suites/rbd/mirror/base/install.yaml rename to qa/suites/rbd/mirror-thrash/base/install.yaml diff --git a/qa/suites/rbd/mirror/cluster/+ b/qa/suites/rbd/mirror-thrash/cluster/+ similarity index 100% rename from qa/suites/rbd/mirror/cluster/+ rename to qa/suites/rbd/mirror-thrash/cluster/+ diff --git a/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml b/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml new file mode 100644 index 0000000000000..700c029dfafc2 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml @@ -0,0 +1,25 @@ +meta: +- desc: 2 ceph clusters with 1 mon and 3 osds each +roles: +- - cluster1.mon.a + - cluster1.mgr.x + - cluster2.mgr.x + - cluster1.osd.0 + - cluster1.osd.1 + - cluster1.osd.2 + - cluster1.client.0 + - cluster2.client.0 +- - cluster2.mon.a + - cluster2.osd.0 + - cluster2.osd.1 + - cluster2.osd.2 + - cluster1.client.mirror + - cluster1.client.mirror.0 + - cluster1.client.mirror.1 + - cluster1.client.mirror.2 + - cluster1.client.mirror.3 + - cluster2.client.mirror + - cluster2.client.mirror.0 + - cluster2.client.mirror.1 + - cluster2.client.mirror.2 + - cluster2.client.mirror.3 diff --git a/qa/suites/rbd/mirror/cluster/openstack.yaml b/qa/suites/rbd/mirror-thrash/cluster/openstack.yaml similarity index 100% rename from qa/suites/rbd/mirror/cluster/openstack.yaml rename to qa/suites/rbd/mirror-thrash/cluster/openstack.yaml diff --git a/qa/suites/rbd/mirror-thrash/msgr-failures b/qa/suites/rbd/mirror-thrash/msgr-failures new file mode 120000 index 0000000000000..db59eb46cbbfa --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/msgr-failures @@ -0,0 +1 @@ +../basic/msgr-failures \ No newline at end of file diff --git a/qa/suites/rbd/mirror-thrash/objectstore b/qa/suites/rbd/mirror-thrash/objectstore new file mode 120000 index 0000000000000..4c8ebadfde871 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/objectstore @@ -0,0 +1 @@ +../../../objectstore \ No newline at end of file diff --git a/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml b/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml new file mode 100644 index 0000000000000..a239c588503ed --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml @@ -0,0 +1,23 @@ +meta: +- desc: run four rbd-mirror daemons per cluster +tasks: +- rbd-mirror: + client: cluster1.client.mirror.0 +- rbd-mirror: + client: cluster1.client.mirror.1 +- rbd-mirror: + client: cluster1.client.mirror.2 +- rbd-mirror: + client: cluster1.client.mirror.3 +- rbd-mirror: + client: cluster2.client.mirror.0 +- rbd-mirror: + client: cluster2.client.mirror.1 +- rbd-mirror: + client: cluster2.client.mirror.2 +- rbd-mirror: + client: cluster2.client.mirror.3 +- rbd-mirror-thrash: + cluster: cluster1 +- rbd-mirror-thrash: + cluster: cluster2 diff --git a/qa/suites/rbd/mirror-thrash/users/mirror.yaml b/qa/suites/rbd/mirror-thrash/users/mirror.yaml new file mode 100644 index 0000000000000..4915122479746 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/users/mirror.yaml @@ -0,0 +1,18 @@ +meta: +- desc: configure the permissions for client.mirror +overrides: + ceph: + conf: + # override to make these names predictable + client.mirror.0: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.1: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.2: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.3: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml similarity index 91% rename from qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml rename to qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml index cdc486411a856..62bda88173b5e 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml +++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml @@ -7,6 +7,7 @@ tasks: env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml new file mode 100644 index 0000000000000..349d3fc838597 --- /dev/null +++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml @@ -0,0 +1,12 @@ +meta: +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon +tasks: +- workunit: + clients: + cluster1.client.mirror: [rbd/rbd_mirror.sh] + env: + # override workunit setting of CEPH_ARGS='--cluster' + CEPH_ARGS: '' + RBD_MIRROR_INSTANCES: '4' + RBD_MIRROR_USE_EXISTING_CLUSTER: '1' + RBD_MIRROR_USE_RBD_MIRROR: '1' diff --git a/qa/suites/rbd/mirror/base b/qa/suites/rbd/mirror/base new file mode 120000 index 0000000000000..8d9546e2102f5 --- /dev/null +++ b/qa/suites/rbd/mirror/base @@ -0,0 +1 @@ +../mirror-thrash/base \ No newline at end of file diff --git a/qa/suites/rbd/mirror/cluster/2-node.yaml b/qa/suites/rbd/mirror/cluster/2-node.yaml index fbc76bda77364..74f9fb3c466f3 100644 --- a/qa/suites/rbd/mirror/cluster/2-node.yaml +++ b/qa/suites/rbd/mirror/cluster/2-node.yaml @@ -3,15 +3,29 @@ meta: roles: - - cluster1.mon.a - cluster1.mgr.x + - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - - cluster2.mgr.x - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror + - cluster1.client.mirror.0 + - cluster1.client.mirror.1 + - cluster1.client.mirror.2 + - cluster1.client.mirror.3 + - cluster1.client.mirror.4 + - cluster1.client.mirror.5 + - cluster1.client.mirror.6 - cluster2.client.mirror + - cluster2.client.mirror.0 + - cluster2.client.mirror.1 + - cluster2.client.mirror.2 + - cluster2.client.mirror.3 + - cluster2.client.mirror.4 + - cluster2.client.mirror.5 + - cluster2.client.mirror.6 diff --git a/qa/suites/rbd/mirror/msgr-failures b/qa/suites/rbd/mirror/msgr-failures index db59eb46cbbfa..728aeab33471a 120000 --- a/qa/suites/rbd/mirror/msgr-failures +++ b/qa/suites/rbd/mirror/msgr-failures @@ -1 +1 @@ -../basic/msgr-failures \ No newline at end of file +../mirror-thrash/msgr-failures \ No newline at end of file diff --git a/qa/suites/rbd/mirror/objectstore b/qa/suites/rbd/mirror/objectstore index 4c8ebadfde871..d751ff121b10b 120000 --- a/qa/suites/rbd/mirror/objectstore +++ b/qa/suites/rbd/mirror/objectstore @@ -1 +1 @@ -../../../objectstore \ No newline at end of file +../mirror-thrash/objectstore \ No newline at end of file diff --git a/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml b/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml deleted file mode 100644 index 1e762a6e3d5b1..0000000000000 --- a/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml +++ /dev/null @@ -1,19 +0,0 @@ -meta: -- desc: run one rbd-mirror daemon per cluster -overrides: - ceph: - conf: - client.mirror: - # override to make these names predictable - admin socket: /var/run/ceph/$cluster-$name.asok - pid file: /var/run/ceph/$cluster-$name.pid -tasks: -- exec: - cluster1.client.mirror: - - ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd' osd 'profile rbd' - cluster2.client.mirror: - - ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd' osd 'profile rbd' -- rbd-mirror: - client: cluster1.client.mirror -- rbd-mirror: - client: cluster2.client.mirror diff --git a/qa/suites/rbd/mirror/users b/qa/suites/rbd/mirror/users new file mode 120000 index 0000000000000..8d9d0d2c336c6 --- /dev/null +++ b/qa/suites/rbd/mirror/users @@ -0,0 +1 @@ +../mirror-thrash/users \ No newline at end of file diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml new file mode 100644 index 0000000000000..a027b2291184c --- /dev/null +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml @@ -0,0 +1,24 @@ +meta: +- desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon +overrides: + ceph: + conf: + # override to make these names predictable + client.mirror.4: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.5: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid + client.mirror.6: + admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok + pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid +tasks: +- workunit: + clients: + cluster1.client.mirror: [rbd/rbd_mirror_ha.sh] + env: + # override workunit setting of CEPH_ARGS='--cluster' + CEPH_ARGS: '' + RBD_MIRROR_USE_EXISTING_CLUSTER: '1' + timeout: 6h diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml index 2e166429f9a59..8ae338d5ea9e5 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml @@ -7,5 +7,5 @@ tasks: env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' - RBD_MIRROR_USE_RBD_MIRROR: '1' diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 233c06dd3f377..ee8044ef239a2 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -10,7 +10,7 @@ . $(dirname $0)/rbd_mirror_helpers.sh testlog "TEST: add image and test replay" -start_mirror ${CLUSTER1} +start_mirrors ${CLUSTER1} image=test create_image ${CLUSTER2} ${POOL} ${image} set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" @@ -27,11 +27,11 @@ compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" testlog "TEST: stop mirror, add image, start mirror and test replay" -stop_mirror ${CLUSTER1} +stop_mirrors ${CLUSTER1} image1=test1 create_image ${CLUSTER2} ${POOL} ${image1} write_image ${CLUSTER2} ${POOL} ${image1} 100 -start_mirror ${CLUSTER1} +start_mirrors ${CLUSTER1} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position' @@ -47,88 +47,90 @@ wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' compare_images ${POOL} ${image} -testlog "TEST: stop/start/restart mirror via admin socket" -admin_daemon ${CLUSTER1} rbd mirror stop -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - -admin_daemon ${CLUSTER1} rbd mirror start -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror restart -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror stop -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - -admin_daemon ${CLUSTER1} rbd mirror restart -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - -admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - -admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - -admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - -admin_daemon ${CLUSTER1} rbd mirror flush -admin_daemon ${CLUSTER1} rbd mirror status +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: stop/start/restart mirror via admin socket" + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror start + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + flush ${CLUSTER1} + all_admin_daemons ${CLUSTER1} rbd mirror status +fi testlog "TEST: test image rename" new_name="${image}_RENAMED" rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' -admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name} -admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} +admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} +admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} testlog "TEST: failover and failback" -start_mirror ${CLUSTER2} +start_mirrors ${CLUSTER2} # demote and promote same cluster demote_image ${CLUSTER2} ${POOL} ${image} @@ -303,14 +305,14 @@ done wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" testlog "TEST: disable mirror while daemon is stopped" -stop_mirror ${CLUSTER1} -stop_mirror ${CLUSTER2} +stop_mirrors ${CLUSTER1} +stop_mirrors ${CLUSTER2} set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' disable_mirror ${CLUSTER2} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi -start_mirror ${CLUSTER1} +start_mirrors ${CLUSTER1} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' @@ -325,21 +327,21 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_ compare_images ${POOL} ${image} testlog "TEST: image resync while replayer is stopped" -admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image} +admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image} +admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image} +admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' compare_images ${POOL} ${image} testlog "TEST: request image resync while daemon is offline" -stop_mirror ${CLUSTER1} +stop_mirrors ${CLUSTER1} request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -start_mirror ${CLUSTER1} +start_mirrors ${CLUSTER1} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} @@ -370,7 +372,7 @@ test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" compare_images ${POOL} ${image} testlog " - disconnected after max_concurrent_object_sets reached" -admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image} +admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})" set_image_meta ${CLUSTER2} ${POOL} ${image} \ @@ -382,7 +384,7 @@ set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_journal_max_concurrent_object_sets 0 testlog " - replay is still stopped (disconnected) after restart" -admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image} +admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' @@ -428,8 +430,11 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain request_resync_image ${CLUSTER1} ${POOL} ${image} image_id wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' -testlog "TEST: no blacklists" -CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries" -CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries" +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + # teuthology will trash the daemon + testlog "TEST: no blacklists" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries" +fi echo OK diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh index fcb8d765129fe..fc08c1dbe6017 100755 --- a/qa/workunits/rbd/rbd_mirror_ha.sh +++ b/qa/workunits/rbd/rbd_mirror_ha.sh @@ -3,6 +3,8 @@ # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode # +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7} + . $(dirname $0)/rbd_mirror_helpers.sh is_leader() @@ -23,7 +25,7 @@ wait_for_leader() for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do sleep $s - for instance in `seq 0 9`; do + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do is_leader ${instance} || continue LEADER=${instance} return 0 diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 4e4c5640c7121..a81d8989c05e3 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions # @@ -18,6 +18,7 @@ # rbd-mirror daemons. # RBD_MIRROR_VARGS - use this to pass additional arguments to vstart.sh # when starting clusters. +# RBD_MIRROR_INSTANCES - number of daemons to start per cluster # # The cleanup can be done as a separate step, running the script with # `cleanup ${RBD_MIRROR_TEMDIR}' arguments. @@ -64,13 +65,18 @@ # ../qa/workunits/rbd/rbd_mirror.sh cleanup # +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2} + CLUSTER1=cluster1 CLUSTER2=cluster2 POOL=mirror PARENT_POOL=mirror_parent TEMPDIR= -USER_ID=mirror -export CEPH_ARGS="--id ${USER_ID}" +CEPH_ID=${CEPH_ID:-mirror} +MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} +export CEPH_ARGS="--id ${CEPH_ID}" + +LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1)) CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src) CEPH_BIN=. @@ -128,11 +134,7 @@ daemon_asok_file() set_cluster_instance "${local_cluster}" local_cluster instance - if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - echo $(ceph-conf --cluster $local_cluster --name "client.${CEPH_ID}" 'admin socket') - else - echo "${TEMPDIR}/rbd-mirror.${local_cluster}_daemon.${instance}.${cluster}.asok" - fi + echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket') } daemon_pid_file() @@ -142,11 +144,7 @@ daemon_pid_file() set_cluster_instance "${cluster}" cluster instance - if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - echo $(ceph-conf --cluster $cluster --name "client.${CEPH_ID}" 'pid file') - else - echo "${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.pid" - fi + echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file') } testlog() @@ -176,6 +174,77 @@ expect_failure() return 0 } +create_users() +{ + local cluster=$1 + + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${CEPH_ID} \ + mon 'profile rbd' osd 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \ + mon 'profile rbd' osd 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + done +} + +update_users() +{ + local cluster=$1 + + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth caps client.${CEPH_ID} \ + mon 'profile rbd' osd 'profile rbd' + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth caps client.${MIRROR_USER_ID_PREFIX}${instance} \ + mon 'profile rbd' osd 'profile rbd' + done +} + +setup_cluster() +{ + local cluster=$1 + + CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS} + + cd ${CEPH_ROOT} + rm -f ${TEMPDIR}/${cluster}.conf + ln -s $(readlink -f run/${cluster}/ceph.conf) \ + ${TEMPDIR}/${cluster}.conf + + cd ${TEMPDIR} + create_users "${cluster}" + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + cat<> ${TEMPDIR}/${cluster}.conf +[client.${MIRROR_USER_ID_PREFIX}${instance}] + admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok + pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid +EOF + done +} + +setup_pools() +{ + local cluster=$1 + local remote_cluster=$2 + + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64 + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64 + + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL} + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL} + + rbd --cluster ${cluster} mirror pool enable ${POOL} pool + rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image + + rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster} + rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster} +} + setup() { local c @@ -191,46 +260,15 @@ setup() fi if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then - cd ${CEPH_ROOT} - CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER1} -n ${RBD_MIRROR_VARGS} - CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER2} -n ${RBD_MIRROR_VARGS} - - CEPH_ARGS='' ceph --conf run/${CLUSTER1}/ceph.conf \ - auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \ - run/${CLUSTER1}/keyring - CEPH_ARGS='' ceph --conf run/${CLUSTER2}/ceph.conf \ - auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \ - run/${CLUSTER2}/keyring - - rm -f ${TEMPDIR}/${CLUSTER1}.conf - ln -s $(readlink -f run/${CLUSTER1}/ceph.conf) \ - ${TEMPDIR}/${CLUSTER1}.conf - rm -f ${TEMPDIR}/${CLUSTER2}.conf - ln -s $(readlink -f run/${CLUSTER2}/ceph.conf) \ - ${TEMPDIR}/${CLUSTER2}.conf - - cd ${TEMPDIR} + setup_cluster "${CLUSTER1}" + setup_cluster "${CLUSTER2}" + else + update_users "${CLUSTER1}" + update_users "${CLUSTER2}" fi - CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64 - CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${PARENT_POOL} 64 64 - CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${PARENT_POOL} 64 64 - CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64 - - CEPH_ARGS='' rbd --cluster ${CLUSTER1} pool init ${POOL} - CEPH_ARGS='' rbd --cluster ${CLUSTER2} pool init ${POOL} - CEPH_ARGS='' rbd --cluster ${CLUSTER1} pool init ${PARENT_POOL} - CEPH_ARGS='' rbd --cluster ${CLUSTER2} pool init ${PARENT_POOL} - - rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool - rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool - rbd --cluster ${CLUSTER1} mirror pool enable ${PARENT_POOL} image - rbd --cluster ${CLUSTER2} mirror pool enable ${PARENT_POOL} image - - rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2} - rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1} - rbd --cluster ${CLUSTER1} mirror pool peer add ${PARENT_POOL} ${CLUSTER2} - rbd --cluster ${CLUSTER2} mirror pool peer add ${PARENT_POOL} ${CLUSTER1} + setup_pools "${CLUSTER1}" "${CLUSTER2}" + setup_pools "${CLUSTER2}" "${CLUSTER1}" } cleanup() @@ -241,9 +279,7 @@ cleanup() set +e for cluster in "${CLUSTER1}" "${CLUSTER2}"; do - for instance in `seq 0 9`; do - stop_mirror "${cluster}:${instance}" - done + stop_mirrors "${cluster}" done if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then @@ -271,10 +307,8 @@ start_mirror() rbd-mirror \ --cluster ${cluster} \ - --id mirror \ - --pid-file=$(daemon_pid_file "${cluster}:${instance}") \ + --id ${MIRROR_USER_ID_PREFIX}${instance} \ --log-file=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log \ - --admin-socket=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.\$cluster.asok \ --rbd-mirror-delete-retry-interval=5 \ --rbd-mirror-image-state-check-interval=5 \ --rbd-mirror-journal-poll-age=1 \ @@ -285,6 +319,15 @@ start_mirror() ${RBD_MIRROR_ARGS} } +start_mirrors() +{ + local cluster=$1 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + start_mirror "${cluster}:${instance}" + done +} + stop_mirror() { local cluster=$1 @@ -308,6 +351,16 @@ stop_mirror() rm -f $(daemon_pid_file "${cluster}") } +stop_mirrors() +{ + local cluster=$1 + local sig=$2 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + stop_mirror "${cluster}:${instance}" "${sig}" + done +} + admin_daemon() { local cluster=$1 ; shift @@ -321,6 +374,30 @@ admin_daemon() ceph --admin-daemon ${asok_file} $@ } +admin_daemons() +{ + local cluster_instance=$1 ; shift + local cluster="${cluster_instance%:*}" + local instance="${cluster_instance##*:}" + + if [ "${instance}" != "${cluster_instance}" ]; then + admin_daemon "${cluster}:${instance}" $@ + else + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${instance}" $@ && return 0 + done + fi +} + +all_admin_daemons() +{ + local cluster=$1 ; shift + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}" $@ + done +} + status() { local cluster daemon image_pool image @@ -357,47 +434,49 @@ status() for cluster in "${CLUSTER1}" "${CLUSTER2}" do - local pid_file=$(daemon_pid_file ${cluster} ) - if [ ! -e ${pid_file} ] - then - echo "${cluster} rbd-mirror not running or unknown" \ - "(${pid_file} not exist)" - continue - fi - - local pid - pid=$(cat ${pid_file} 2>/dev/null) || : - if [ -z "${pid}" ] - then - echo "${cluster} rbd-mirror not running or unknown" \ - "(can't find pid using ${pid_file})" - ret=1 - continue - fi - - echo "${daemon} rbd-mirror process in ps output:" - if ps auxww | + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + local pid_file=$(daemon_pid_file ${cluster}:${instance}) + if [ ! -e ${pid_file} ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(${pid_file} not exist)" + continue + fi + + local pid + pid=$(cat ${pid_file} 2>/dev/null) || : + if [ -z "${pid}" ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(can't find pid using ${pid_file})" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror process in ps output:" + if ps auxww | awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' - then + then + echo + echo "${cluster} rbd-mirror not running" \ + "(can't find pid $pid in ps output)" + ret=1 + continue + fi echo - echo "${cluster} rbd-mirror not running" \ - "(can't find pid $pid in ps output)" - ret=1 - continue - fi - echo - local asok_file=$(daemon_asok_file ${cluster} ${cluster}) - if [ ! -S "${asok_file}" ] - then - echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" - ret=1 - continue - fi + local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) + if [ ! -S "${asok_file}" ] + then + echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" + ret=1 + continue + fi - echo "${cluster} rbd-mirror status" - ceph --admin-daemon ${asok_file} rbd mirror status - echo + echo "${cluster} rbd-mirror status" + ceph --admin-daemon ${asok_file} rbd mirror status + echo + done done return ${ret} @@ -415,7 +494,11 @@ flush() cmd="${cmd} ${pool}/${image}" fi - admin_daemon "${cluster}" ${cmd} + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + admin_daemons "${cluster}" ${cmd} && return 0 + done + return 1 } test_image_replay_state() @@ -426,12 +509,8 @@ test_image_replay_state() local test_state=$4 local current_state=stopped - admin_daemon "${cluster}" help | - fgrep "\"rbd mirror status ${pool}/${image}\"" && - admin_daemon "${cluster}" rbd mirror status ${pool}/${image} | - grep -i 'state.*Replaying' && - current_state=started - + admin_daemons "${cluster}" rbd mirror status ${pool}/${image} | + grep -i 'state.*Replaying' && current_state=started test "${test_state}" = "${current_state}" } diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index b07bf0ed118bf..f21984eeee448 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -83,8 +83,8 @@ wait_for_pool_healthy() return 1 } -start_mirror ${CLUSTER1} -start_mirror ${CLUSTER2} +start_mirrors ${CLUSTER1} +start_mirrors ${CLUSTER2} testlog "TEST: add image and test replay after client crashes" image=test -- 2.39.5