qa/suites/rbd: thrash rbd-mirror daemon during tests

Fixes: http://tracker.ceph.com/issues/18753
Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/qa/suites/rbd/mirror-ha/% b/qa/suites/rbd/mirror-ha/%
deleted file mode 100644 (file)
index e69de29..0000000

diff --git a/qa/suites/rbd/mirror-ha/base b/qa/suites/rbd/mirror-ha/base
deleted file mode 120000 (symlink)
index 09e88d4..0000000
--- a/qa/suites/rbd/mirror-ha/base
+++ /dev/null
@@ -1 +0,0 @@
-../mirror/base
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-ha/cluster b/qa/suites/rbd/mirror-ha/cluster
deleted file mode 120000 (symlink)
index 47e95a2..0000000
--- a/qa/suites/rbd/mirror-ha/cluster
+++ /dev/null
@@ -1 +0,0 @@
-../mirror/cluster
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-ha/msgr-failures b/qa/suites/rbd/mirror-ha/msgr-failures
deleted file mode 120000 (symlink)
index c0ae027..0000000
--- a/qa/suites/rbd/mirror-ha/msgr-failures
+++ /dev/null
@@ -1 +0,0 @@
-../mirror/msgr-failures
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-ha/objectstore b/qa/suites/rbd/mirror-ha/objectstore
deleted file mode 120000 (symlink)
index 1a5e90e..0000000
--- a/qa/suites/rbd/mirror-ha/objectstore
+++ /dev/null
@@ -1 +0,0 @@
-../mirror/objectstore
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml b/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml
deleted file mode 100644 (file)
index 406318f..0000000
--- a/qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-meta:
-- desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon
-tasks:
-- exec:
-    cluster1.client.mirror:
-    - ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
-    cluster2.client.mirror:
-    - ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
-- workunit:
-    clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_ha.sh]
-    env:
-      # override workunit setting of CEPH_ARGS='--cluster'
-      CEPH_ARGS: ''
-      RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
-    timeout: 6h

diff --git a/qa/suites/rbd/mirror-thrash/% b/qa/suites/rbd/mirror-thrash/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rbd/mirror-thrash/base/install.yaml b/qa/suites/rbd/mirror-thrash/base/install.yaml
new file mode 100644 (file)
index 0000000..365c3a8
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/base/install.yaml
@@ -0,0 +1,9 @@
+meta:
+- desc: run two ceph clusters and install rbd-mirror
+tasks:
+- install:
+    extra_packages: [rbd-mirror]
+- ceph:
+    cluster: cluster1
+- ceph:
+    cluster: cluster2

diff --git a/qa/suites/rbd/mirror-thrash/cluster/+ b/qa/suites/rbd/mirror-thrash/cluster/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml b/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml
new file mode 100644 (file)
index 0000000..700c029
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/cluster/2-node.yaml
@@ -0,0 +1,25 @@
+meta:
+- desc: 2 ceph clusters with 1 mon and 3 osds each
+roles:
+- - cluster1.mon.a
+  - cluster1.mgr.x
+  - cluster2.mgr.x
+  - cluster1.osd.0
+  - cluster1.osd.1
+  - cluster1.osd.2
+  - cluster1.client.0
+  - cluster2.client.0
+- - cluster2.mon.a
+  - cluster2.osd.0
+  - cluster2.osd.1
+  - cluster2.osd.2
+  - cluster1.client.mirror
+  - cluster1.client.mirror.0
+  - cluster1.client.mirror.1
+  - cluster1.client.mirror.2
+  - cluster1.client.mirror.3
+  - cluster2.client.mirror
+  - cluster2.client.mirror.0
+  - cluster2.client.mirror.1
+  - cluster2.client.mirror.2
+  - cluster2.client.mirror.3

diff --git a/qa/suites/rbd/mirror-thrash/cluster/openstack.yaml b/qa/suites/rbd/mirror-thrash/cluster/openstack.yaml
new file mode 100644 (file)
index 0000000..f4d1349
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/cluster/openstack.yaml
@@ -0,0 +1,4 @@
+openstack:
+  - volumes: # attached to each instance
+      count: 3
+      size: 30 # GB

diff --git a/qa/suites/rbd/mirror-thrash/msgr-failures b/qa/suites/rbd/mirror-thrash/msgr-failures
new file mode 120000 (symlink)
index 0000000..db59eb4
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/msgr-failures
@@ -0,0 +1 @@
+../basic/msgr-failures
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-thrash/objectstore b/qa/suites/rbd/mirror-thrash/objectstore
new file mode 120000 (symlink)
index 0000000..4c8ebad
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/objectstore
@@ -0,0 +1 @@
+../../../objectstore
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml b/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml
new file mode 100644 (file)
index 0000000..a239c58
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/rbd-mirror/four-per-cluster.yaml
@@ -0,0 +1,23 @@
+meta:
+- desc: run four rbd-mirror daemons per cluster
+tasks:
+- rbd-mirror:
+    client: cluster1.client.mirror.0
+- rbd-mirror:
+    client: cluster1.client.mirror.1
+- rbd-mirror:
+    client: cluster1.client.mirror.2
+- rbd-mirror:
+    client: cluster1.client.mirror.3
+- rbd-mirror:
+    client: cluster2.client.mirror.0
+- rbd-mirror:
+    client: cluster2.client.mirror.1
+- rbd-mirror:
+    client: cluster2.client.mirror.2
+- rbd-mirror:
+    client: cluster2.client.mirror.3
+- rbd-mirror-thrash:
+    cluster: cluster1
+- rbd-mirror-thrash:
+    cluster: cluster2

diff --git a/qa/suites/rbd/mirror-thrash/users/mirror.yaml b/qa/suites/rbd/mirror-thrash/users/mirror.yaml
new file mode 100644 (file)
index 0000000..4915122
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/users/mirror.yaml
@@ -0,0 +1,18 @@
+meta:
+- desc: configure the permissions for client.mirror
+overrides:
+  ceph:
+    conf:
+      # override to make these names predictable
+      client.mirror.0:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.1:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.2:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.3:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid

diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml
new file mode 100644 (file)
index 0000000..62bda88
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-stress-workunit.yaml
@@ -0,0 +1,13 @@
+meta:
+- desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror: [rbd/rbd_mirror_stress.sh]
+    env:
+      # override workunit setting of CEPH_ARGS='--cluster'
+      CEPH_ARGS: ''
+      RBD_MIRROR_INSTANCES: '4'
+      RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
+      RBD_MIRROR_USE_RBD_MIRROR: '1'
+    timeout: 6h

diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml
new file mode 100644 (file)
index 0000000..349d3fc
--- /dev/null
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-workunit.yaml
@@ -0,0 +1,12 @@
+meta:
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
+    env:
+      # override workunit setting of CEPH_ARGS='--cluster'
+      CEPH_ARGS: ''
+      RBD_MIRROR_INSTANCES: '4'
+      RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
+      RBD_MIRROR_USE_RBD_MIRROR: '1'

diff --git a/qa/suites/rbd/mirror/base b/qa/suites/rbd/mirror/base
new file mode 120000 (symlink)
index 0000000..8d9546e
--- /dev/null
+++ b/qa/suites/rbd/mirror/base
@@ -0,0 +1 @@
+../mirror-thrash/base
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror/base/install.yaml b/qa/suites/rbd/mirror/base/install.yaml
deleted file mode 100644 (file)
index 365c3a8..0000000
--- a/qa/suites/rbd/mirror/base/install.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-meta:
-- desc: run two ceph clusters and install rbd-mirror
-tasks:
-- install:
-    extra_packages: [rbd-mirror]
-- ceph:
-    cluster: cluster1
-- ceph:
-    cluster: cluster2

diff --git a/qa/suites/rbd/mirror/cluster/+ b/qa/suites/rbd/mirror/cluster/+
deleted file mode 100644 (file)
index e69de29..0000000

diff --git a/qa/suites/rbd/mirror/cluster/2-node.yaml b/qa/suites/rbd/mirror/cluster/2-node.yaml
index fbc76bda77364a7d35e6d25b4b3f1e639e7bebe6..74f9fb3c466f36014ed408c6fcc939fe4e9e9e17 100644 (file)
--- a/qa/suites/rbd/mirror/cluster/2-node.yaml
+++ b/qa/suites/rbd/mirror/cluster/2-node.yaml
@@ -3,15 +3,29 @@ meta:
 roles:
 - - cluster1.mon.a
   - cluster1.mgr.x
+  - cluster2.mgr.x
   - cluster1.osd.0
   - cluster1.osd.1
   - cluster1.osd.2
   - cluster1.client.0
   - cluster2.client.0
 - - cluster2.mon.a
-  - cluster2.mgr.x
   - cluster2.osd.0
   - cluster2.osd.1
   - cluster2.osd.2
   - cluster1.client.mirror
+  - cluster1.client.mirror.0
+  - cluster1.client.mirror.1
+  - cluster1.client.mirror.2
+  - cluster1.client.mirror.3
+  - cluster1.client.mirror.4
+  - cluster1.client.mirror.5
+  - cluster1.client.mirror.6
   - cluster2.client.mirror
+  - cluster2.client.mirror.0
+  - cluster2.client.mirror.1
+  - cluster2.client.mirror.2
+  - cluster2.client.mirror.3
+  - cluster2.client.mirror.4
+  - cluster2.client.mirror.5
+  - cluster2.client.mirror.6

diff --git a/qa/suites/rbd/mirror/cluster/openstack.yaml b/qa/suites/rbd/mirror/cluster/openstack.yaml
deleted file mode 100644 (file)
index f4d1349..0000000
--- a/qa/suites/rbd/mirror/cluster/openstack.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-openstack:
-  - volumes: # attached to each instance
-      count: 3
-      size: 30 # GB

diff --git a/qa/suites/rbd/mirror/msgr-failures b/qa/suites/rbd/mirror/msgr-failures
index db59eb46cbbfa895e5e91d611a5fc46d22e2a10c..728aeab33471afe3892292ac249a7c9d6c3a394f 120000 (symlink)
--- a/qa/suites/rbd/mirror/msgr-failures
+++ b/qa/suites/rbd/mirror/msgr-failures
@@ -1 +1 @@
-../basic/msgr-failures
\ No newline at end of file
+../mirror-thrash/msgr-failures
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror/objectstore b/qa/suites/rbd/mirror/objectstore
index 4c8ebadfde871b95effbdd015b4e07bea9a7e2fa..d751ff121b10b89fb4d2115b24b8a86d7995b7fe 120000 (symlink)
--- a/qa/suites/rbd/mirror/objectstore
+++ b/qa/suites/rbd/mirror/objectstore
@@ -1 +1 @@
-../../../objectstore
\ No newline at end of file
+../mirror-thrash/objectstore
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml b/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml
deleted file mode 100644 (file)
index 1e762a6..0000000
--- a/qa/suites/rbd/mirror/rbd-mirror/one-per-cluster.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-meta:
-- desc: run one rbd-mirror daemon per cluster
-overrides:
-  ceph:
-    conf:
-      client.mirror:
-        # override to make these names predictable
-        admin socket: /var/run/ceph/$cluster-$name.asok
-        pid file: /var/run/ceph/$cluster-$name.pid
-tasks:
-- exec:
-    cluster1.client.mirror:
-    - ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
-    cluster2.client.mirror:
-    - ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
-- rbd-mirror:
-    client: cluster1.client.mirror
-- rbd-mirror:
-    client: cluster2.client.mirror

diff --git a/qa/suites/rbd/mirror/users b/qa/suites/rbd/mirror/users
new file mode 120000 (symlink)
index 0000000..8d9d0d2
--- /dev/null
+++ b/qa/suites/rbd/mirror/users
@@ -0,0 +1 @@
+../mirror-thrash/users
\ No newline at end of file

diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml
new file mode 100644 (file)
index 0000000..a027b22
--- /dev/null
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-ha-workunit.yaml
@@ -0,0 +1,24 @@
+meta:
+- desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon
+overrides:
+  ceph:
+    conf:
+      # override to make these names predictable
+      client.mirror.4:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.5:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.6:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror: [rbd/rbd_mirror_ha.sh]
+    env:
+      # override workunit setting of CEPH_ARGS='--cluster'
+      CEPH_ARGS: ''
+      RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
+    timeout: 6h

diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml
deleted file mode 100644 (file)
index cdc4864..0000000
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon
-tasks:
-- workunit:
-    clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_stress.sh]
-    env:
-      # override workunit setting of CEPH_ARGS='--cluster'
-      CEPH_ARGS: ''
-      RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
-      RBD_MIRROR_USE_RBD_MIRROR: '1'
-    timeout: 6h

diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml
index 2e166429f9a59af61800a42fbfee51cfe972d10d..8ae338d5ea9e5617d81bb38c429e50af12bdb866 100644 (file)
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml
@@ -7,5 +7,5 @@ tasks:
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
-      RBD_MIRROR_USE_RBD_MIRROR: '1'

diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
index 233c06dd3f377c4ed44dafae6ecf0cbf173380e9..ee8044ef239a2709948c66224e3cec87f689f7cb 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror.sh
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -10,7 +10,7 @@
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 testlog "TEST: add image and test replay"
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 image=test
 create_image ${CLUSTER2} ${POOL} ${image}
 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
@@ -27,11 +27,11 @@ compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
 
 testlog "TEST: stop mirror, add image, start mirror and test replay"
-stop_mirror ${CLUSTER1}
+stop_mirrors ${CLUSTER1}
 image1=test1
 create_image ${CLUSTER2} ${POOL} ${image1}
 write_image ${CLUSTER2} ${POOL} ${image1} 100
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
@@ -47,88 +47,90 @@ wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 compare_images ${POOL} ${image}
 
-testlog "TEST: stop/start/restart mirror via admin socket"
-admin_daemon ${CLUSTER1} rbd mirror stop
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-admin_daemon ${CLUSTER1} rbd mirror start
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror restart
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror stop
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-admin_daemon ${CLUSTER1} rbd mirror restart
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-admin_daemon ${CLUSTER1} rbd mirror flush
-admin_daemon ${CLUSTER1} rbd mirror status
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  testlog "TEST: stop/start/restart mirror via admin socket"
+  all_admin_daemons ${CLUSTER1} rbd mirror stop
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror start
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  flush ${CLUSTER1}
+  all_admin_daemons ${CLUSTER1} rbd mirror status
+fi
 
 testlog "TEST: test image rename"
 new_name="${image}_RENAMED"
 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
-admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
+admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 testlog "TEST: failover and failback"
-start_mirror ${CLUSTER2}
+start_mirrors ${CLUSTER2}
 
 # demote and promote same cluster
 demote_image ${CLUSTER2} ${POOL} ${image}
@@ -303,14 +305,14 @@ done
 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 
 testlog "TEST: disable mirror while daemon is stopped"
-stop_mirror ${CLUSTER1}
-stop_mirror ${CLUSTER2}
+stop_mirrors ${CLUSTER1}
+stop_mirrors ${CLUSTER2}
 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 disable_mirror ${CLUSTER2} ${POOL} ${image}
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 fi
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
@@ -325,21 +327,21 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_
 compare_images ${POOL} ${image}
 
 testlog "TEST: image resync while replayer is stopped"
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 compare_images ${POOL} ${image}
 
 testlog "TEST: request image resync while daemon is offline"
-stop_mirror ${CLUSTER1}
+stop_mirrors ${CLUSTER1}
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
@@ -370,7 +372,7 @@ test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 compare_images ${POOL} ${image}
 
 testlog " - disconnected after max_concurrent_object_sets reached"
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 set_image_meta ${CLUSTER2} ${POOL} ${image} \
@@ -382,7 +384,7 @@ set_image_meta ${CLUSTER2} ${POOL} ${image} \
               conf_rbd_journal_max_concurrent_object_sets 0
 
 testlog " - replay is still stopped (disconnected) after restart"
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 
@@ -428,8 +430,11 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 
-testlog "TEST: no blacklists"
-CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
-CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  # teuthology will trash the daemon
+  testlog "TEST: no blacklists"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+fi
 
 echo OK

diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh
index fcb8d765129fe5fc444d6f3eceb8fdf30c3b76c3..fc08c1dbe601766111b300eab250303c59266ab8 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_ha.sh
+++ b/qa/workunits/rbd/rbd_mirror_ha.sh
@@ -3,6 +3,8 @@
 # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode
 #
 
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7}
+
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 is_leader()
@@ -23,7 +25,7 @@ wait_for_leader()
 
     for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do
        sleep $s
-       for instance in `seq 0 9`; do
+       for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
            is_leader ${instance} || continue
            LEADER=${instance}
            return 0

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 4e4c5640c712137a26c44682313057e1b7b0c250..a81d8989c05e35f65037c20586933b0fbdee7b72 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -x
 #
 # rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions
 #
@@ -18,6 +18,7 @@
 #                         rbd-mirror daemons.
 #  RBD_MIRROR_VARGS     - use this to pass additional arguments to vstart.sh
 #                         when starting clusters.
+#  RBD_MIRROR_INSTANCES - number of daemons to start per cluster
 #
 # The cleanup can be done as a separate step, running the script with
 # `cleanup ${RBD_MIRROR_TEMDIR}' arguments.
@@ -64,13 +65,18 @@
 #     ../qa/workunits/rbd/rbd_mirror.sh cleanup
 #
 
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2}
+
 CLUSTER1=cluster1
 CLUSTER2=cluster2
 POOL=mirror
 PARENT_POOL=mirror_parent
 TEMPDIR=
-USER_ID=mirror
-export CEPH_ARGS="--id ${USER_ID}"
+CEPH_ID=${CEPH_ID:-mirror}
+MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.}
+export CEPH_ARGS="--id ${CEPH_ID}"
+
+LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1))
 
 CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src)
 CEPH_BIN=.
@@ -128,11 +134,7 @@ daemon_asok_file()
 
     set_cluster_instance "${local_cluster}" local_cluster instance
 
-    if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-        echo $(ceph-conf --cluster $local_cluster --name "client.${CEPH_ID}" 'admin socket')
-    else
-        echo "${TEMPDIR}/rbd-mirror.${local_cluster}_daemon.${instance}.${cluster}.asok"
-    fi
+    echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket')
 }
 
 daemon_pid_file()
@@ -142,11 +144,7 @@ daemon_pid_file()
 
     set_cluster_instance "${cluster}" cluster instance
 
-    if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-        echo $(ceph-conf --cluster $cluster --name "client.${CEPH_ID}" 'pid file')
-    else
-        echo "${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.pid"
-    fi
+    echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file')
 }
 
 testlog()
@@ -176,6 +174,77 @@ expect_failure()
     return 0
 }
 
+create_users()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ceph --cluster "${cluster}" \
+        auth get-or-create client.${CEPH_ID} \
+        mon 'profile rbd' osd 'profile rbd' >> \
+        ${CEPH_ROOT}/run/${cluster}/keyring
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        CEPH_ARGS='' ceph --cluster "${cluster}" \
+            auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \
+            mon 'profile rbd' osd 'profile rbd' >> \
+            ${CEPH_ROOT}/run/${cluster}/keyring
+    done
+}
+
+update_users()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ceph --cluster "${cluster}" \
+        auth caps client.${CEPH_ID} \
+        mon 'profile rbd' osd 'profile rbd'
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        CEPH_ARGS='' ceph --cluster "${cluster}" \
+            auth caps client.${MIRROR_USER_ID_PREFIX}${instance} \
+            mon 'profile rbd' osd 'profile rbd'
+    done
+}
+
+setup_cluster()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS}
+
+    cd ${CEPH_ROOT}
+    rm -f ${TEMPDIR}/${cluster}.conf
+    ln -s $(readlink -f run/${cluster}/ceph.conf) \
+       ${TEMPDIR}/${cluster}.conf
+
+    cd ${TEMPDIR}
+    create_users "${cluster}"
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        cat<<EOF >> ${TEMPDIR}/${cluster}.conf
+[client.${MIRROR_USER_ID_PREFIX}${instance}]
+    admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok
+    pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid
+EOF
+    done
+}
+
+setup_pools()
+{
+    local cluster=$1
+    local remote_cluster=$2
+
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64
+
+    CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL}
+    CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL}
+
+    rbd --cluster ${cluster} mirror pool enable ${POOL} pool
+    rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image
+
+    rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster}
+    rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster}
+}
+
 setup()
 {
     local c
@@ -191,46 +260,15 @@ setup()
     fi
 
     if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
-        cd ${CEPH_ROOT}
-        CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER1} -n ${RBD_MIRROR_VARGS}
-        CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER2} -n ${RBD_MIRROR_VARGS}
-
-        CEPH_ARGS='' ceph --conf run/${CLUSTER1}/ceph.conf \
-            auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \
-            run/${CLUSTER1}/keyring
-        CEPH_ARGS='' ceph --conf run/${CLUSTER2}/ceph.conf \
-            auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \
-            run/${CLUSTER2}/keyring
-
-       rm -f ${TEMPDIR}/${CLUSTER1}.conf
-        ln -s $(readlink -f run/${CLUSTER1}/ceph.conf) \
-           ${TEMPDIR}/${CLUSTER1}.conf
-       rm -f ${TEMPDIR}/${CLUSTER2}.conf
-        ln -s $(readlink -f run/${CLUSTER2}/ceph.conf) \
-           ${TEMPDIR}/${CLUSTER2}.conf
-
-        cd ${TEMPDIR}
+       setup_cluster "${CLUSTER1}"
+       setup_cluster "${CLUSTER2}"
+    else
+       update_users "${CLUSTER1}"
+       update_users "${CLUSTER2}"
     fi
 
-    CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64
-    CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${PARENT_POOL} 64 64
-    CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${PARENT_POOL} 64 64
-    CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64
-
-    CEPH_ARGS='' rbd --cluster ${CLUSTER1} pool init ${POOL}
-    CEPH_ARGS='' rbd --cluster ${CLUSTER2} pool init ${POOL}
-    CEPH_ARGS='' rbd --cluster ${CLUSTER1} pool init ${PARENT_POOL}
-    CEPH_ARGS='' rbd --cluster ${CLUSTER2} pool init ${PARENT_POOL}
-
-    rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool
-    rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool
-    rbd --cluster ${CLUSTER1} mirror pool enable ${PARENT_POOL} image
-    rbd --cluster ${CLUSTER2} mirror pool enable ${PARENT_POOL} image
-
-    rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2}
-    rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1}
-    rbd --cluster ${CLUSTER1} mirror pool peer add ${PARENT_POOL} ${CLUSTER2}
-    rbd --cluster ${CLUSTER2} mirror pool peer add ${PARENT_POOL} ${CLUSTER1}
+    setup_pools "${CLUSTER1}" "${CLUSTER2}"
+    setup_pools "${CLUSTER2}" "${CLUSTER1}"
 }
 
 cleanup()
@@ -241,9 +279,7 @@ cleanup()
     set +e
 
     for cluster in "${CLUSTER1}" "${CLUSTER2}"; do
-       for instance in `seq 0 9`; do
-           stop_mirror "${cluster}:${instance}"
-       done
+       stop_mirrors "${cluster}"
     done
 
     if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
@@ -271,10 +307,8 @@ start_mirror()
 
     rbd-mirror \
        --cluster ${cluster} \
-        --id mirror \
-       --pid-file=$(daemon_pid_file "${cluster}:${instance}") \
+        --id ${MIRROR_USER_ID_PREFIX}${instance} \
        --log-file=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log \
-       --admin-socket=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.\$cluster.asok \
        --rbd-mirror-delete-retry-interval=5 \
        --rbd-mirror-image-state-check-interval=5 \
        --rbd-mirror-journal-poll-age=1 \
@@ -285,6 +319,15 @@ start_mirror()
        ${RBD_MIRROR_ARGS}
 }
 
+start_mirrors()
+{
+    local cluster=$1
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        start_mirror "${cluster}:${instance}"
+    done
+}
+
 stop_mirror()
 {
     local cluster=$1
@@ -308,6 +351,16 @@ stop_mirror()
     rm -f $(daemon_pid_file "${cluster}")
 }
 
+stop_mirrors()
+{
+    local cluster=$1
+    local sig=$2
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        stop_mirror "${cluster}:${instance}" "${sig}"
+    done
+}
+
 admin_daemon()
 {
     local cluster=$1 ; shift
@@ -321,6 +374,30 @@ admin_daemon()
     ceph --admin-daemon ${asok_file} $@
 }
 
+admin_daemons()
+{
+    local cluster_instance=$1 ; shift
+    local cluster="${cluster_instance%:*}"
+    local instance="${cluster_instance##*:}"
+
+    if [ "${instance}" != "${cluster_instance}" ]; then
+       admin_daemon "${cluster}:${instance}" $@
+    else
+        for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+           admin_daemon "${cluster}:${instance}" $@ && return 0
+        done
+    fi
+}
+
+all_admin_daemons()
+{
+    local cluster=$1 ; shift
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+       admin_daemon "${cluster}" $@
+    done
+}
+
 status()
 {
     local cluster daemon image_pool image
@@ -357,47 +434,49 @@ status()
 
     for cluster in "${CLUSTER1}" "${CLUSTER2}"
     do
-       local pid_file=$(daemon_pid_file ${cluster} )
-       if [ ! -e ${pid_file} ]
-       then
-           echo "${cluster} rbd-mirror not running or unknown" \
-                "(${pid_file} not exist)"
-           continue
-       fi
-
-       local pid
-       pid=$(cat ${pid_file} 2>/dev/null) || :
-       if [ -z "${pid}" ]
-       then
-           echo "${cluster} rbd-mirror not running or unknown" \
-                "(can't find pid using ${pid_file})"
-           ret=1
-           continue
-       fi
-
-       echo "${daemon} rbd-mirror process in ps output:"
-       if ps auxww |
+       for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+           local pid_file=$(daemon_pid_file ${cluster}:${instance})
+           if [ ! -e ${pid_file} ]
+           then
+               echo "${cluster} rbd-mirror not running or unknown" \
+                    "(${pid_file} not exist)"
+               continue
+           fi
+
+           local pid
+           pid=$(cat ${pid_file} 2>/dev/null) || :
+           if [ -z "${pid}" ]
+           then
+               echo "${cluster} rbd-mirror not running or unknown" \
+                    "(can't find pid using ${pid_file})"
+               ret=1
+               continue
+           fi
+
+           echo "${daemon} rbd-mirror process in ps output:"
+           if ps auxww |
                awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
-       then
+           then
+               echo
+               echo "${cluster} rbd-mirror not running" \
+                    "(can't find pid $pid in ps output)"
+               ret=1
+               continue
+           fi
            echo
-           echo "${cluster} rbd-mirror not running" \
-                "(can't find pid $pid in ps output)"
-           ret=1
-           continue
-       fi
-       echo
 
-       local asok_file=$(daemon_asok_file ${cluster} ${cluster})
-       if [ ! -S "${asok_file}" ]
-       then
-           echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
-           ret=1
-           continue
-       fi
+           local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster})
+           if [ ! -S "${asok_file}" ]
+           then
+               echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
+               ret=1
+               continue
+           fi
 
-       echo "${cluster} rbd-mirror status"
-       ceph --admin-daemon ${asok_file} rbd mirror status
-       echo
+           echo "${cluster} rbd-mirror status"
+           ceph --admin-daemon ${asok_file} rbd mirror status
+           echo
+       done
     done
 
     return ${ret}
@@ -415,7 +494,11 @@ flush()
        cmd="${cmd} ${pool}/${image}"
     fi
 
-    admin_daemon "${cluster}" ${cmd}
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+       sleep ${s}
+       admin_daemons "${cluster}" ${cmd} && return 0
+    done
+    return 1
 }
 
 test_image_replay_state()
@@ -426,12 +509,8 @@ test_image_replay_state()
     local test_state=$4
     local current_state=stopped
 
-    admin_daemon "${cluster}" help |
-       fgrep "\"rbd mirror status ${pool}/${image}\"" &&
-    admin_daemon "${cluster}" rbd mirror status ${pool}/${image} |
-       grep -i 'state.*Replaying' &&
-    current_state=started
-
+    admin_daemons "${cluster}" rbd mirror status ${pool}/${image} |
+       grep -i 'state.*Replaying' && current_state=started
     test "${test_state}" = "${current_state}"
 }
 

diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
index b07bf0ed118bfe6d334b7f16a735631e10fa273f..f21984eeee448deed9f3cbb1732c344c11024b4b 100755 (executable)
--- a/qa/workunits/rbd/rbd_mirror_stress.sh
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -83,8 +83,8 @@ wait_for_pool_healthy()
     return 1
 }
 
-start_mirror ${CLUSTER1}
-start_mirror ${CLUSTER2}
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
 
 testlog "TEST: add image and test replay after client crashes"
 image=test