test: add cephfs-mirror HA active/active workunit and test yamls

Fixes: http://tracker.ceph.com/issues/50372
Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/qa/suites/fs/mirror-ha/% b/qa/suites/fs/mirror-ha/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/mirror-ha/.qa b/qa/suites/fs/mirror-ha/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/begin.yaml b/qa/suites/fs/mirror-ha/begin.yaml
new file mode 120000 (symlink)
index 0000000..311d404
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/begin.yaml
@@ -0,0 +1 @@
+.qa/cephfs/begin.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml
new file mode 100644 (file)
index 0000000..095f089
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml
@@ -0,0 +1,12 @@
+meta:
+- desc: run one cephfs-mirror daemon on primary cluster
+tasks:
+- cephfs-mirror:
+    client: client.mirror1
+    run_in_foreground: True
+- cephfs-mirror:
+    client: client.mirror2
+    run_in_foreground: True
+- cephfs-mirror:
+    client: client.mirror3
+    run_in_foreground: True

diff --git a/qa/suites/fs/mirror-ha/clients/+ b/qa/suites/fs/mirror-ha/clients/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/mirror-ha/clients/.qa b/qa/suites/fs/mirror-ha/clients/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/clients/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/clients/mirror.yaml b/qa/suites/fs/mirror-ha/clients/mirror.yaml
new file mode 100644 (file)
index 0000000..620c821
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/clients/mirror.yaml
@@ -0,0 +1,32 @@
+meta:
+- desc: configure the permissions for client.mirror
+overrides:
+  ceph:
+    conf:
+      client:
+        debug cephfs_mirror: 20
+        log to stderr: false
+      # make these predictable
+      client.mirror1:
+        admin socket: /var/run/ceph/cephfs-mirror1.asok
+        pid file: /var/run/ceph/cephfs-mirror1.pid
+      client.mirror2:
+        admin socket: /var/run/ceph/cephfs-mirror2.asok
+        pid file: /var/run/ceph/cephfs-mirror2.pid
+      client.mirror3:
+        admin socket: /var/run/ceph/cephfs-mirror3.asok
+        pid file: /var/run/ceph/cephfs-mirror3.pid
+tasks:
+- exec:
+    client.mirror1:
+      - "sudo ceph auth caps client.mirror1 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'"
+    client.mirror2:
+      - "sudo ceph auth caps client.mirror2 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'"
+    client.mirror3:
+      - "sudo ceph auth caps client.mirror3 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'"
+    client.mirror_remote:
+      - "sudo ceph auth caps client.mirror_remote mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'"
+    client.1:
+      - "sudo ceph auth caps client.0 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'"
+    client.2:
+      - "sudo ceph auth caps client.1 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'"

diff --git a/qa/suites/fs/mirror-ha/cluster/+ b/qa/suites/fs/mirror-ha/cluster/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/mirror-ha/cluster/1-node.yaml b/qa/suites/fs/mirror-ha/cluster/1-node.yaml
new file mode 100644 (file)
index 0000000..cc70c10
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/cluster/1-node.yaml
@@ -0,0 +1,20 @@
+meta:
+- desc: 1 ceph cluster with 1 mon, 1 mgr, 3 osds, 5 mdss
+roles:
+- - mon.a
+  - mgr.x
+  - mds.a
+  - mds.b
+  - mds.c
+  - mds.d
+  - mds.e
+  - osd.0
+  - osd.1
+  - osd.2
+  - client.0
+  - client.1
+  - client.2
+  - client.mirror1
+  - client.mirror2
+  - client.mirror3
+  - client.mirror_remote
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/objectstore/.qa b/qa/suites/fs/mirror-ha/objectstore/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/objectstore/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml
new file mode 120000 (symlink)
index 0000000..a59cf51
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+.qa/objectstore/bluestore-bitmap.yaml
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/overrides/+ b/qa/suites/fs/mirror-ha/overrides/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/fs/mirror-ha/overrides/.qa b/qa/suites/fs/mirror-ha/overrides/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/overrides/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml
new file mode 100644 (file)
index 0000000..d40fa4c
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml
@@ -0,0 +1,14 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(FS_DEGRADED\)
+      - \(MDS_FAILED\)
+      - \(MDS_DEGRADED\)
+      - \(FS_WITH_FAILED_MDS\)
+      - \(MDS_DAMAGE\)
+      - \(MDS_ALL_DOWN\)
+      - \(MDS_UP_LESS_THAN_MAX\)
+      - \(FS_INLINE_DATA_DEPRECATED\)
+      - Reduced data availability
+      - Degraded data redundancy

diff --git a/qa/suites/fs/mirror-ha/supported-random-distro$ b/qa/suites/fs/mirror-ha/supported-random-distro$
new file mode 120000 (symlink)
index 0000000..0862b44
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/supported-random-distro$
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/workloads/.qa b/qa/suites/fs/mirror-ha/workloads/.qa
new file mode 120000 (symlink)
index 0000000..fea2489
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/workloads/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file

diff --git a/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml
new file mode 100644 (file)
index 0000000..18eb46d
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml
@@ -0,0 +1,37 @@
+meta:
+- desc: run the cephfs_mirror_ha.sh workunit to test cephfs-mirror daemon in HA active/active mode
+
+overrides:
+  ceph:
+    conf:
+      mgr:
+        debug client: 10
+
+tasks:
+  - exec:
+      client.1:
+        - "ceph fs volume create dc"
+        - "ceph fs volume create dc-backup"
+  - ceph-fuse:
+      client.1:
+        cephfs_name: dc
+      client.2:
+        cephfs_name: dc-backup
+  - cephfs_mirror_thrash:
+      randomize: False
+      max_thrash_delay: 10
+  - workunit:
+      subdir: mirror
+      cleanup: False
+      clients:
+        client.1: [fs/cephfs_mirror_ha_gen.sh]
+      timeout: 1h
+  - exec:
+      client.2:
+        - "echo verifying synchronized snapshots..."
+  - workunit:
+      subdir: mirror
+      cleanup: False
+      clients:
+        client.2: [fs/cephfs_mirror_ha_verify.sh]
+      timeout: 3h
\ No newline at end of file

diff --git a/qa/suites/fs/mirror/tasks/mirror.yaml b/qa/suites/fs/mirror/tasks/mirror.yaml
index af60495e53f8b9b05cc1efe1eafe969b3b23aaa7..07c1e24efbf8e766d5f4b5a809fafd1c8f17ad2c 100644 (file)
--- a/qa/suites/fs/mirror/tasks/mirror.yaml
+++ b/qa/suites/fs/mirror/tasks/mirror.yaml
@@ -7,4 +7,4 @@ overrides:
 tasks:
   - cephfs_test_runner:
       modules:
-        - tasks.cephfs.test_mirroring
+        - tasks.cephfs.test_mirroring.TestMirroring

diff --git a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml
index af60495e53f8b9b05cc1efe1eafe969b3b23aaa7..07c1e24efbf8e766d5f4b5a809fafd1c8f17ad2c 100644 (file)
--- a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml
+++ b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml
@@ -7,4 +7,4 @@ overrides:
 tasks:
   - cephfs_test_runner:
       modules:
-        - tasks.cephfs.test_mirroring
+        - tasks.cephfs.test_mirroring.TestMirroring

diff --git a/qa/workunits/fs/cephfs_mirror_ha_gen.sh b/qa/workunits/fs/cephfs_mirror_ha_gen.sh
new file mode 100755 (executable)
index 0000000..35ee9d4
--- /dev/null
+++ b/qa/workunits/fs/cephfs_mirror_ha_gen.sh
@@ -0,0 +1,69 @@
+#!/bin/bash -ex
+#
+# cephfs_mirror_ha_gen.sh - generate workload to synchronize
+#
+
+. $(dirname $0)/cephfs_mirror_helpers.sh
+
+cleanup()
+{
+    for i in `seq 1 $NR_DIRECTORIES`
+    do
+        local repo_name="${REPO_PATH_PFX}_$i"
+        for j in `seq 1 $NR_SNAPSHOTS`
+        do
+            snap_name=$repo_name/.snap/snap_$j
+            if test -d $snap_name; then
+                rmdir $snap_name
+            fi
+        done
+    done
+    exit 1
+}
+trap cleanup EXIT
+
+configure_peer()
+{
+    ceph mgr module enable mirroring
+    ceph fs snapshot mirror enable $PRIMARY_FS
+    ceph fs snapshot mirror peer_add $PRIMARY_FS client.mirror_remote@ceph $BACKUP_FS
+
+    for i in `seq 1 $NR_DIRECTORIES`
+    do
+        local repo_name="${REPO_PATH_PFX}_$i"
+        ceph fs snapshot mirror add $PRIMARY_FS "$MIRROR_SUBDIR/$repo_name"
+    done
+}
+
+create_snaps()
+{
+    for i in `seq 1 $NR_DIRECTORIES`
+    do
+        local repo_name="${REPO_PATH_PFX}_$i"
+        for j in `seq 1 $NR_SNAPSHOTS`
+        do
+            snap_name=$repo_name/.snap/snap_$j
+            r=$(( $RANDOM % 100 + 5 ))
+            arr=($repo_name "reset" "--hard" "HEAD~$r")
+            exec_git_cmd "${arr[@]}"
+            mkdir $snap_name
+            store_checksum $snap_name
+        done
+    done
+}
+
+unset CEPH_CLI_TEST_DUP_COMMAND
+
+echo "running generator on prmary file system..."
+
+# setup git repos to be used as data set
+setup_repos
+
+# turn on mirroring, add peers...
+configure_peer
+
+# snapshots on primary
+create_snaps
+
+# do not cleanup when exiting on success..
+trap - EXIT

diff --git a/qa/workunits/fs/cephfs_mirror_ha_verify.sh b/qa/workunits/fs/cephfs_mirror_ha_verify.sh
new file mode 100755 (executable)
index 0000000..8d8b385
--- /dev/null
+++ b/qa/workunits/fs/cephfs_mirror_ha_verify.sh
@@ -0,0 +1,40 @@
+#!/bin/bash -ex
+#
+# cephfs_mirror_ha_verify.sh - verify synchronized snapshots
+#
+
+. $(dirname $0)/cephfs_mirror_helpers.sh
+
+echo "running verifier on secondary file system..."
+
+for i in `seq 1 $NR_DIRECTORIES`
+do
+    repo_name="${REPO_PATH_PFX}_$i"
+    for j in `seq 1 $NR_SNAPSHOTS`
+    do
+        for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 64 128 128
+        do
+            sleep $s
+            snap_name=$repo_name/.snap/snap_$j
+            if test -d $repo_name; then
+                echo "checking snapshot [$snap_name] in $repo_name"
+                if test -d $snap_name; then
+                    echo "generating hash for $snap_name"
+                    cksum=''
+                    calc_checksum $snap_name cksum
+                    ret=$(compare_checksum $cksum $snap_name)
+                    if [ $ret -ne 0 ]; then
+                        echo "checksum failed $snap_name ($cksum)"
+                        return $ret
+                    else
+                        echo "checksum matched $snap_name ($cksum)"
+                        break
+                    fi
+                fi
+            fi
+        done
+        echo "couldn't complete verification for: $snap_name"
+    done
+done
+
+echo "verify done!"

diff --git a/qa/workunits/fs/cephfs_mirror_helpers.sh b/qa/workunits/fs/cephfs_mirror_helpers.sh
new file mode 100644 (file)
index 0000000..69f1c6f
--- /dev/null
+++ b/qa/workunits/fs/cephfs_mirror_helpers.sh
@@ -0,0 +1,66 @@
+PRIMARY_FS='dc'
+BACKUP_FS='dc-backup'
+
+REPO=ceph-qa-suite
+REPO_DIR=ceph_repo
+REPO_PATH_PFX="$REPO_DIR/$REPO"
+
+NR_DIRECTORIES=4
+NR_SNAPSHOTS=4
+MIRROR_SUBDIR='/mirror'
+
+calc_checksum()
+{
+    local path=$1
+    local -n ref=$2
+    ref=`find -L $path -type f -exec md5sum {} +  | awk '{ print $1 }' | md5sum | awk '{ print $1 }'`
+}
+
+store_checksum()
+{
+    local path=$1
+    local cksum='' #something invalid
+    local fhash=`echo -n $path | md5sum | awk '{ print $1 }'`
+    calc_checksum $path cksum
+    echo -n $cksum > "/tmp/primary-$fhash"
+}
+
+compare_checksum()
+{
+    local ret=0
+    local cksum=$1
+    local path=$2
+    local fhash=`echo -n $path | md5sum | awk '{ print $1 }'`
+    local cksum_ondisk=`cat /tmp/primary-$fhash`
+    if [ $cksum != $cksum_ondisk ]; then
+        echo "$cksum <> $cksum_ondisk"
+        ret=1
+    fi
+    echo $ret
+}
+
+exec_git_cmd()
+{
+    local arg=("$@")
+    local repo_name=${arg[0]}
+    local cmd=${arg[@]:1}
+    git --git-dir "$repo_name/.git" $cmd
+}
+
+clone_repo()
+{
+    local repo_name=$1
+    git clone --branch giant "http://github.com/ceph/$REPO" $repo_name
+}
+
+setup_repos()
+{
+    mkdir "$REPO_DIR"
+
+    for i in `seq 1 $NR_DIRECTORIES`
+    do
+        local repo_name="${REPO_PATH_PFX}_$i"
+        mkdir $repo_name
+        clone_repo $repo_name
+    done
+}