From e7bdf7fb35e57c01e05d24e2f3ec6d6c7fcdcf0a Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 16 Jun 2021 01:23:27 -0400 Subject: [PATCH] test: add cephfs-mirror HA active/active workunit and test yamls Fixes: http://tracker.ceph.com/issues/50372 Signed-off-by: Venky Shankar --- qa/suites/fs/mirror-ha/% | 0 qa/suites/fs/mirror-ha/.qa | 1 + qa/suites/fs/mirror-ha/begin.yaml | 1 + .../cephfs-mirror/three-per-cluster.yaml | 12 ++++ qa/suites/fs/mirror-ha/clients/+ | 0 qa/suites/fs/mirror-ha/clients/.qa | 1 + qa/suites/fs/mirror-ha/clients/mirror.yaml | 32 +++++++++ qa/suites/fs/mirror-ha/cluster/+ | 0 qa/suites/fs/mirror-ha/cluster/1-node.yaml | 20 ++++++ qa/suites/fs/mirror-ha/objectstore/.qa | 1 + .../objectstore/bluestore-bitmap.yaml | 1 + qa/suites/fs/mirror-ha/overrides/+ | 0 qa/suites/fs/mirror-ha/overrides/.qa | 1 + .../mirror-ha/overrides/whitelist_health.yaml | 14 ++++ .../fs/mirror-ha/supported-random-distro$ | 1 + qa/suites/fs/mirror-ha/workloads/.qa | 1 + .../workloads/cephfs-mirror-ha-workunit.yaml | 37 ++++++++++ qa/suites/fs/mirror/tasks/mirror.yaml | 2 +- .../fs/valgrind/mirror/tasks/mirror.yaml | 2 +- qa/workunits/fs/cephfs_mirror_ha_gen.sh | 69 +++++++++++++++++++ qa/workunits/fs/cephfs_mirror_ha_verify.sh | 40 +++++++++++ qa/workunits/fs/cephfs_mirror_helpers.sh | 66 ++++++++++++++++++ 22 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 qa/suites/fs/mirror-ha/% create mode 120000 qa/suites/fs/mirror-ha/.qa create mode 120000 qa/suites/fs/mirror-ha/begin.yaml create mode 100644 qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml create mode 100644 qa/suites/fs/mirror-ha/clients/+ create mode 120000 qa/suites/fs/mirror-ha/clients/.qa create mode 100644 qa/suites/fs/mirror-ha/clients/mirror.yaml create mode 100644 qa/suites/fs/mirror-ha/cluster/+ create mode 100644 qa/suites/fs/mirror-ha/cluster/1-node.yaml create mode 120000 qa/suites/fs/mirror-ha/objectstore/.qa create mode 120000 qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml create mode 100644 qa/suites/fs/mirror-ha/overrides/+ create mode 120000 qa/suites/fs/mirror-ha/overrides/.qa create mode 100644 qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml create mode 120000 qa/suites/fs/mirror-ha/supported-random-distro$ create mode 120000 qa/suites/fs/mirror-ha/workloads/.qa create mode 100644 qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml create mode 100755 qa/workunits/fs/cephfs_mirror_ha_gen.sh create mode 100755 qa/workunits/fs/cephfs_mirror_ha_verify.sh create mode 100644 qa/workunits/fs/cephfs_mirror_helpers.sh diff --git a/qa/suites/fs/mirror-ha/% b/qa/suites/fs/mirror-ha/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/.qa b/qa/suites/fs/mirror-ha/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/begin.yaml b/qa/suites/fs/mirror-ha/begin.yaml new file mode 120000 index 0000000000000..311d404f7c276 --- /dev/null +++ b/qa/suites/fs/mirror-ha/begin.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin.yaml \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml new file mode 100644 index 0000000000000..095f0893ac26a --- /dev/null +++ b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml @@ -0,0 +1,12 @@ +meta: +- desc: run one cephfs-mirror daemon on primary cluster +tasks: +- cephfs-mirror: + client: client.mirror1 + run_in_foreground: True +- cephfs-mirror: + client: client.mirror2 + run_in_foreground: True +- cephfs-mirror: + client: client.mirror3 + run_in_foreground: True diff --git a/qa/suites/fs/mirror-ha/clients/+ b/qa/suites/fs/mirror-ha/clients/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/clients/.qa b/qa/suites/fs/mirror-ha/clients/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/clients/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/clients/mirror.yaml b/qa/suites/fs/mirror-ha/clients/mirror.yaml new file mode 100644 index 0000000000000..620c821e1170e --- /dev/null +++ b/qa/suites/fs/mirror-ha/clients/mirror.yaml @@ -0,0 +1,32 @@ +meta: +- desc: configure the permissions for client.mirror +overrides: + ceph: + conf: + client: + debug cephfs_mirror: 20 + log to stderr: false + # make these predictable + client.mirror1: + admin socket: /var/run/ceph/cephfs-mirror1.asok + pid file: /var/run/ceph/cephfs-mirror1.pid + client.mirror2: + admin socket: /var/run/ceph/cephfs-mirror2.asok + pid file: /var/run/ceph/cephfs-mirror2.pid + client.mirror3: + admin socket: /var/run/ceph/cephfs-mirror3.asok + pid file: /var/run/ceph/cephfs-mirror3.pid +tasks: +- exec: + client.mirror1: + - "sudo ceph auth caps client.mirror1 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror2: + - "sudo ceph auth caps client.mirror2 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror3: + - "sudo ceph auth caps client.mirror3 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror_remote: + - "sudo ceph auth caps client.mirror_remote mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" + client.1: + - "sudo ceph auth caps client.0 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" + client.2: + - "sudo ceph auth caps client.1 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" diff --git a/qa/suites/fs/mirror-ha/cluster/+ b/qa/suites/fs/mirror-ha/cluster/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/cluster/1-node.yaml b/qa/suites/fs/mirror-ha/cluster/1-node.yaml new file mode 100644 index 0000000000000..cc70c106d67e4 --- /dev/null +++ b/qa/suites/fs/mirror-ha/cluster/1-node.yaml @@ -0,0 +1,20 @@ +meta: +- desc: 1 ceph cluster with 1 mon, 1 mgr, 3 osds, 5 mdss +roles: +- - mon.a + - mgr.x + - mds.a + - mds.b + - mds.c + - mds.d + - mds.e + - osd.0 + - osd.1 + - osd.2 + - client.0 + - client.1 + - client.2 + - client.mirror1 + - client.mirror2 + - client.mirror3 + - client.mirror_remote \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/objectstore/.qa b/qa/suites/fs/mirror-ha/objectstore/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/objectstore/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml new file mode 120000 index 0000000000000..a59cf5175069a --- /dev/null +++ b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/objectstore/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/overrides/+ b/qa/suites/fs/mirror-ha/overrides/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/overrides/.qa b/qa/suites/fs/mirror-ha/overrides/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/overrides/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml new file mode 100644 index 0000000000000..d40fa4cb82ce7 --- /dev/null +++ b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_DEGRADED\) + - \(MDS_FAILED\) + - \(MDS_DEGRADED\) + - \(FS_WITH_FAILED_MDS\) + - \(MDS_DAMAGE\) + - \(MDS_ALL_DOWN\) + - \(MDS_UP_LESS_THAN_MAX\) + - \(FS_INLINE_DATA_DEPRECATED\) + - Reduced data availability + - Degraded data redundancy diff --git a/qa/suites/fs/mirror-ha/supported-random-distro$ b/qa/suites/fs/mirror-ha/supported-random-distro$ new file mode 120000 index 0000000000000..0862b4457b37a --- /dev/null +++ b/qa/suites/fs/mirror-ha/supported-random-distro$ @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$ \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/workloads/.qa b/qa/suites/fs/mirror-ha/workloads/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/workloads/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml new file mode 100644 index 0000000000000..18eb46d786ffb --- /dev/null +++ b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml @@ -0,0 +1,37 @@ +meta: +- desc: run the cephfs_mirror_ha.sh workunit to test cephfs-mirror daemon in HA active/active mode + +overrides: + ceph: + conf: + mgr: + debug client: 10 + +tasks: + - exec: + client.1: + - "ceph fs volume create dc" + - "ceph fs volume create dc-backup" + - ceph-fuse: + client.1: + cephfs_name: dc + client.2: + cephfs_name: dc-backup + - cephfs_mirror_thrash: + randomize: False + max_thrash_delay: 10 + - workunit: + subdir: mirror + cleanup: False + clients: + client.1: [fs/cephfs_mirror_ha_gen.sh] + timeout: 1h + - exec: + client.2: + - "echo verifying synchronized snapshots..." + - workunit: + subdir: mirror + cleanup: False + clients: + client.2: [fs/cephfs_mirror_ha_verify.sh] + timeout: 3h \ No newline at end of file diff --git a/qa/suites/fs/mirror/tasks/mirror.yaml b/qa/suites/fs/mirror/tasks/mirror.yaml index af60495e53f8b..07c1e24efbf8e 100644 --- a/qa/suites/fs/mirror/tasks/mirror.yaml +++ b/qa/suites/fs/mirror/tasks/mirror.yaml @@ -7,4 +7,4 @@ overrides: tasks: - cephfs_test_runner: modules: - - tasks.cephfs.test_mirroring + - tasks.cephfs.test_mirroring.TestMirroring diff --git a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml index af60495e53f8b..07c1e24efbf8e 100644 --- a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml +++ b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml @@ -7,4 +7,4 @@ overrides: tasks: - cephfs_test_runner: modules: - - tasks.cephfs.test_mirroring + - tasks.cephfs.test_mirroring.TestMirroring diff --git a/qa/workunits/fs/cephfs_mirror_ha_gen.sh b/qa/workunits/fs/cephfs_mirror_ha_gen.sh new file mode 100755 index 0000000000000..35ee9d4c7ee34 --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_gen.sh @@ -0,0 +1,69 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_gen.sh - generate workload to synchronize +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +cleanup() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + if test -d $snap_name; then + rmdir $snap_name + fi + done + done + exit 1 +} +trap cleanup EXIT + +configure_peer() +{ + ceph mgr module enable mirroring + ceph fs snapshot mirror enable $PRIMARY_FS + ceph fs snapshot mirror peer_add $PRIMARY_FS client.mirror_remote@ceph $BACKUP_FS + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + ceph fs snapshot mirror add $PRIMARY_FS "$MIRROR_SUBDIR/$repo_name" + done +} + +create_snaps() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + r=$(( $RANDOM % 100 + 5 )) + arr=($repo_name "reset" "--hard" "HEAD~$r") + exec_git_cmd "${arr[@]}" + mkdir $snap_name + store_checksum $snap_name + done + done +} + +unset CEPH_CLI_TEST_DUP_COMMAND + +echo "running generator on prmary file system..." + +# setup git repos to be used as data set +setup_repos + +# turn on mirroring, add peers... +configure_peer + +# snapshots on primary +create_snaps + +# do not cleanup when exiting on success.. +trap - EXIT diff --git a/qa/workunits/fs/cephfs_mirror_ha_verify.sh b/qa/workunits/fs/cephfs_mirror_ha_verify.sh new file mode 100755 index 0000000000000..8d8b3859cb6ba --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_verify.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_verify.sh - verify synchronized snapshots +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +echo "running verifier on secondary file system..." + +for i in `seq 1 $NR_DIRECTORIES` +do + repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 64 128 128 + do + sleep $s + snap_name=$repo_name/.snap/snap_$j + if test -d $repo_name; then + echo "checking snapshot [$snap_name] in $repo_name" + if test -d $snap_name; then + echo "generating hash for $snap_name" + cksum='' + calc_checksum $snap_name cksum + ret=$(compare_checksum $cksum $snap_name) + if [ $ret -ne 0 ]; then + echo "checksum failed $snap_name ($cksum)" + return $ret + else + echo "checksum matched $snap_name ($cksum)" + break + fi + fi + fi + done + echo "couldn't complete verification for: $snap_name" + done +done + +echo "verify done!" diff --git a/qa/workunits/fs/cephfs_mirror_helpers.sh b/qa/workunits/fs/cephfs_mirror_helpers.sh new file mode 100644 index 0000000000000..69f1c6f3d77e0 --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_helpers.sh @@ -0,0 +1,66 @@ +PRIMARY_FS='dc' +BACKUP_FS='dc-backup' + +REPO=ceph-qa-suite +REPO_DIR=ceph_repo +REPO_PATH_PFX="$REPO_DIR/$REPO" + +NR_DIRECTORIES=4 +NR_SNAPSHOTS=4 +MIRROR_SUBDIR='/mirror' + +calc_checksum() +{ + local path=$1 + local -n ref=$2 + ref=`find -L $path -type f -exec md5sum {} + | awk '{ print $1 }' | md5sum | awk '{ print $1 }'` +} + +store_checksum() +{ + local path=$1 + local cksum='' #something invalid + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + calc_checksum $path cksum + echo -n $cksum > "/tmp/primary-$fhash" +} + +compare_checksum() +{ + local ret=0 + local cksum=$1 + local path=$2 + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + local cksum_ondisk=`cat /tmp/primary-$fhash` + if [ $cksum != $cksum_ondisk ]; then + echo "$cksum <> $cksum_ondisk" + ret=1 + fi + echo $ret +} + +exec_git_cmd() +{ + local arg=("$@") + local repo_name=${arg[0]} + local cmd=${arg[@]:1} + git --git-dir "$repo_name/.git" $cmd +} + +clone_repo() +{ + local repo_name=$1 + git clone --branch giant "http://github.com/ceph/$REPO" $repo_name +} + +setup_repos() +{ + mkdir "$REPO_DIR" + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + mkdir $repo_name + clone_repo $repo_name + done +} -- 2.39.5