From 9486e95db9be21fc8943421d32131b9edb9d2bb9 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 16 Jun 2023 14:01:52 +0200 Subject: [PATCH] qa/workunits/rbd: make continuous export-diff test actually work The current version is pretty useless: - "rbd bench" writes the same byte (0xff) over and over again, so almost all checksumming is in vain - snapshots are taken in a steady state (i.e. not under I/O), so no race conditions can get exposed - even with these caveats, it's not wired up into the suite Redo this workunit to be a reliable reproducer for the issue fixed in the previous commit and wire it up for both krbd and rbd-nbd. Signed-off-by: Ilya Dryomov (cherry picked from commit acb270a3dd07211f06aa11f8792a333f3a1b796d) --- .../workloads/krbd_diff_continuous.yaml | 12 ++ .../workloads/rbd_nbd_diff_continuous.yaml | 14 ++ qa/workunits/rbd/diff_continuous.sh | 142 ++++++++++++------ 3 files changed, 120 insertions(+), 48 deletions(-) create mode 100644 qa/suites/krbd/thrash/workloads/krbd_diff_continuous.yaml create mode 100644 qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml diff --git a/qa/suites/krbd/thrash/workloads/krbd_diff_continuous.yaml b/qa/suites/krbd/thrash/workloads/krbd_diff_continuous.yaml new file mode 100644 index 0000000000000..5907718d508fd --- /dev/null +++ b/qa/suites/krbd/thrash/workloads/krbd_diff_continuous.yaml @@ -0,0 +1,12 @@ +overrides: + install: + ceph: + extra_system_packages: + - pv +tasks: +- workunit: + clients: + all: + - rbd/diff_continuous.sh + env: + RBD_DEVICE_TYPE: "krbd" diff --git a/qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml b/qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml new file mode 100644 index 0000000000000..e0a7ebe335a07 --- /dev/null +++ b/qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml @@ -0,0 +1,14 @@ +overrides: + install: + ceph: + extra_packages: + - rbd-nbd + extra_system_packages: + - pv +tasks: +- workunit: + clients: + client.0: + - rbd/diff_continuous.sh + env: + RBD_DEVICE_TYPE: "nbd" diff --git a/qa/workunits/rbd/diff_continuous.sh b/qa/workunits/rbd/diff_continuous.sh index b8f7e8b7e1375..fd1785e07c4b4 100755 --- a/qa/workunits/rbd/diff_continuous.sh +++ b/qa/workunits/rbd/diff_continuous.sh @@ -1,60 +1,106 @@ #!/usr/bin/env bash + set -ex +set -o pipefail + +function untar_workload() { + local i + for ((i = 0; i < 10; i++)); do + pv -L 10M linux-5.4.tar.gz > "${MOUNT}/linux-5.4.tar.gz" + tar -C "${MOUNT}" -xzf "${MOUNT}/linux-5.4.tar.gz" + sync "${MOUNT}" + rm -rf "${MOUNT}"/linux-5.4* + done +} + +function check_object_map() { + local spec="$1" + + rbd object-map check "${spec}" -max=20 -size=1500 - -iosize=16384 -iototal=16384000 -iothreads=16 - -parent=`uuidgen`"-parent" -src=`uuidgen`"-src"; -dst=`uuidgen`"-dst"; - -function cleanup() { - rbd snap purge $src || : - rbd rm $src || : - rbd snap purge $dst || : - rbd rm $dst || : - rbd snap unprotect $parent --snap parent || : - rbd snap purge $parent || : - rbd rm $parent || : + local flags + flags="$(rbd info "${spec}" | grep 'flags: ')" + if [[ "${flags}" =~ object\ map\ invalid ]]; then + echo "Object map invalid at ${spec}" + exit 1 + fi + if [[ "${flags}" =~ fast\ diff\ invalid ]]; then + echo "Fast diff invalid at ${spec}" + exit 1 + fi } -trap cleanup EXIT - -# start from a clone -rbd create $parent --size $size --image-format 2 --stripe-count 8 --stripe-unit 65536 -rbd bench --io-type write $parent --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand -rbd snap create $parent --snap parent -rbd snap protect $parent --snap parent -rbd clone $parent@parent $src --stripe-count 4 --stripe-unit 262144 -rbd create $dst --size $size --image-format 2 --order 19 - -# mirror for a while -for s in `seq 1 $max`; do - rbd snap create $src --snap=snap$s - rbd export-diff $src@snap$s - $lastsnap | rbd import-diff - $dst & - rbd bench --io-type write $src --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand & - wait - lastsnap="--from-snap snap$s" + +# RBD_DEVICE_TYPE is intended to be set from yaml, default to krbd +readonly DEVICE_TYPE="${RBD_DEVICE_TYPE:-krbd}" + +BASE_UUID="$(uuidgen)" +readonly BASE_UUID + +readonly SIZE="2G" +readonly SRC="${BASE_UUID}-src" +readonly DST="${BASE_UUID}-dst" +readonly MOUNT="${BASE_UUID}-mnt" + +rbd create -s "${SIZE}" --stripe-unit 64K --stripe-count 8 \ + --image-feature exclusive-lock,object-map,fast-diff "${SRC}" +rbd create -s "${SIZE}" --object-size 512K "${DST}" + +dev="$(sudo rbd device map -t "${DEVICE_TYPE}" "${SRC}")" +sudo mkfs.ext4 "${dev}" +mkdir "${MOUNT}" +sudo mount "${dev}" "${MOUNT}" +sudo chown "$(whoami)" "${MOUNT}" + +# start untar in the background +wget https://download.ceph.com/qa/linux-5.4.tar.gz +untar_workload & +untar_pid=$! + +# export initial incremental +snap_num=1 +rbd snap create "${SRC}@snap${snap_num}" +rbd export-diff "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff" + +# keep exporting successive incrementals while untar is running +while kill -0 "${untar_pid}"; do + snap_num=$((snap_num + 1)) + rbd snap create "${SRC}@snap${snap_num}" + sleep $((RANDOM % 4 + 1)) + rbd export-diff --whole-object --from-snap "snap$((snap_num - 1))" \ + "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff" done -#trap "" EXIT -#exit 0 +sudo umount "${MOUNT}" +sudo rbd device unmap -t "${DEVICE_TYPE}" "${dev}" + +if ! wait "${untar_pid}"; then + echo "untar_workload failed" + exit 1 +fi + +echo "Exported ${snap_num} incrementals" +if ((snap_num < 30)); then + echo "Too few incrementals" + exit 1 +fi # validate -for s in `seq 1 $max`; do - ssum=`rbd export $src@snap$s - | md5sum` - dsum=`rbd export $dst@snap$s - | md5sum` - if [ "$ssum" != "$dsum" ]; then - echo different sum at snap$s - exit 1 +for ((i = 1; i <= snap_num; i++)); do + rbd import-diff "${BASE_UUID}@snap${i}.diff" "${DST}" + src_sum="$(rbd export "${SRC}@snap${i}" - | md5sum | awk '{print $1}')" + dst_sum="$(rbd export "${DST}@snap${i}" - | md5sum | awk '{print $1}')" + if [[ "${src_sum}" != "${dst_sum}" ]]; then + echo "Mismatch at snap${i}: ${src_sum} != ${dst_sum}" + exit 1 fi + check_object_map "${SRC}@snap${i}" + # FIXME: this reproduces http://tracker.ceph.com/issues/37876 + # there is no fstrim involved but "rbd import-diff" can produce + # write-zeroes requests which turn into discards under the hood + # actual: EXISTS, expected: EXISTS_CLEAN inconsistency is harmless + # from a data integrity POV and data is validated above regardless, + # so just waive it for now + #check_object_map "${DST}@snap${i}" done -cleanup -trap "" EXIT - echo OK - -- 2.39.5