From: Mykola Golub Date: Sat, 13 Feb 2016 07:29:07 +0000 (+0200) Subject: test: rbd-mirror: add tool to test ImageReplayer X-Git-Tag: v10.1.0~343^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cc4b73b3f539ec5692c1dbb27e57a70e14a7cf4b;p=ceph.git test: rbd-mirror: add tool to test ImageReplayer Signed-off-by: Mykola Golub --- diff --git a/qa/workunits/rbd/rbd_mirror_image_replay.sh b/qa/workunits/rbd/rbd_mirror_image_replay.sh new file mode 100755 index 000000000000..1bc8dd53bd7d --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_image_replay.sh @@ -0,0 +1,156 @@ +#!/bin/sh -xe + +LOC_POOL=rbd_mirror_local$$ +RMT_POOL=rbd_mirror_remote$$ +IMAGE=rbdimagereplay$$ +CLUSTER_ID= +RBD_IMAGE_REPLAY_PID_FILE= +TEMPDIR= + +# +# Functions +# + +setup() +{ + trap cleanup INT TERM EXIT + + TEMPDIR=`mktemp -d` + + CLUSTER_ID=`ceph-conf fsid` + + ceph osd pool create ${LOC_POOL} 128 128 || : + ceph osd pool create ${RMT_POOL} 128 128 || : + + rbd -p ${RMT_POOL} create \ + --image-feature exclusive-lock --image-feature journaling \ + --size 128 ${IMAGE} + + rbd -p ${RMT_POOL} info ${IMAGE} +} + +cleanup() +{ + set +e + + stop_replay + + if [ -n "${RBD_IMAGE_REPLAY_NOCLEANUP}" ] + then + return + fi + + rm -Rf ${TEMPDIR} + remove_image ${LOC_POOL} ${IMAGE} + remove_image ${RMT_POOL} ${IMAGE} + ceph osd pool delete ${LOC_POOL} ${LOC_POOL} --yes-i-really-really-mean-it + ceph osd pool delete ${RMT_POOL} ${RMT_POOL} --yes-i-really-really-mean-it +} + +remove_image() +{ + local pool=$1 + local image=$2 + + if rbd -p ${pool} status ${image} 2>/dev/null; then + for s in 0.1 0.2 0.4 0.8 1.6 3.2 6.4 12.8; do + sleep $s + rbd -p ${pool} status ${image} | grep 'Watchers: none' && break + done + rbd -p ${pool} remove ${image} + fi +} + +start_replay() +{ + RBD_IMAGE_REPLAY_PID_FILE=${TEMPDIR}/rbd-mirror-image-replay.pid + + ceph_test_rbd_mirror_image_replay \ + --pid-file=${RBD_IMAGE_REPLAY_PID_FILE} \ + --log-file=${TEMPDIR}/rbd-mirror-image-replay.log \ + --admin-socket=${TEMPDIR}/rbd-mirror-image-replay.asok \ + --debug-rbd=30 --debug-journaler=30 \ + --debug-rbd_mirror=30 \ + --daemonize=true \ + ${LOC_POOL} ${RMT_POOL} ${IMAGE} +} + +stop_replay() +{ + if [ -z "${RBD_IMAGE_REPLAY_PID_FILE}" ] + then + return 0 + fi + + local pid + pid=$(cat ${RBD_IMAGE_REPLAY_PID_FILE} 2>/dev/null) || : + if [ -n "${pid}" ] + then + kill ${pid} + fi + for s in 0.2 0.4 0.8 1.6 2 4 8; do + sleep $s + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break + done + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' + rm -f ${TEMPDIR}/rbd-mirror-image-replay.asok + rm -f ${RBD_IMAGE_REPLAY_PID_FILE} + RBD_IMAGE_REPLAY_PID_FILE= +} + +wait_for_replay_complete() +{ + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8; do + sleep ${s} + local status_log=${TEMPDIR}/${RMT_POOL}-${IMAGE}.status + rbd -p ${RMT_POOL} journal status --image ${IMAGE} | tee ${status_log} + local master_pos=`sed -nEe 's/^.*id=,.*entry_tid=([0-9]+).*$/\1/p' ${status_log}` + local mirror_pos=`sed -nEe 's/^.*id='${CLUSTER_ID}',.*entry_tid=([0-9]+).*$/\1/p' ${status_log}` + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + done + return 1 +} + +compare_images() +{ + local rmt_export=${TEMPDIR}/${RMT_POOL}-${IMAGE}.export + local loc_export=${TEMPDIR}/${LOC_POOL}-${IMAGE}.export + + rm -f ${rmt_export} ${loc_export} + rbd -p ${RMT_POOL} export ${IMAGE} ${rmt_export} + rbd -p ${LOC_POOL} export ${IMAGE} ${loc_export} + cmp ${rmt_export} ${loc_export} +} + +# +# Main +# + +setup + +start_replay +wait_for_replay_complete +stop_replay +compare_images + +count=32 +rbd -p ${RMT_POOL} bench-write ${IMAGE} --io-size 4096 --io-threads 1 \ + --io-total $((4096 * count)) --io-pattern seq +start_replay +wait_for_replay_complete +compare_images + +rbd -p ${RMT_POOL} bench-write ${IMAGE} --io-size 4096 --io-threads 1 \ + --io-total $((4096 * count)) --io-pattern rand +wait_for_replay_complete +compare_images + +stop_replay + +rbd -p ${RMT_POOL} bench-write ${IMAGE} --io-size 4096 --io-threads 1 \ + --io-total $((4096 * count)) --io-pattern rand +start_replay +wait_for_replay_complete +compare_images + +echo OK diff --git a/src/test/Makefile-client.am b/src/test/Makefile-client.am index 2403f832532b..987e773d04e9 100644 --- a/src/test/Makefile-client.am +++ b/src/test/Makefile-client.am @@ -477,6 +477,22 @@ ceph_test_rbd_mirror_LDADD = \ $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) bin_DEBUGPROGRAMS += ceph_test_rbd_mirror +ceph_test_rbd_mirror_image_replay_SOURCES = \ + test/rbd_mirror/image_replay.cc +ceph_test_rbd_mirror_image_replay_LDADD = \ + librbd_mirror_internal.la \ + librbd_internal.la \ + librbd_api.la \ + $(LIBRBD_TYPES) \ + libjournal.la \ + $(LIBRADOS) $(LIBOSDC) \ + librados_internal.la \ + libcls_rbd_client.la \ + libcls_lock_client.la \ + libcls_journal_client.la \ + $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_rbd_mirror_image_replay + if LINUX ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.cc ceph_test_librbd_fsx_LDADD = \ diff --git a/src/test/rbd_mirror/image_replay.cc b/src/test/rbd_mirror/image_replay.cc new file mode 100644 index 000000000000..8044c139e5fe --- /dev/null +++ b/src/test/rbd_mirror/image_replay.cc @@ -0,0 +1,201 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/ceph_argparse.h" +#include "common/config.h" +#include "common/debug.h" +#include "common/errno.h" +#include "global/global_init.h" +#include "global/signal_handler.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "tools/rbd_mirror/ImageReplayer.h" + +#include +#include + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd-mirror-image-replay: " + +rbd::mirror::ImageReplayer *replayer = nullptr; + +void usage() { + std::cout << "usage: ceph_test_rbd_mirror_image_replay [options...] \\" << std::endl; + std::cout << " " << std::endl; + std::cout << std::endl; + std::cout << " local-pool local (secondary, destination) pool" << std::endl; + std::cout << " remote-pool remote (primary, source) pool" << std::endl; + std::cout << " image image to replay (mirror)" << std::endl; + std::cout << std::endl; + std::cout << "options:\n"; + std::cout << " -m monaddress[:port] connect to specified monitor\n"; + std::cout << " --keyring= path to keyring for local cluster\n"; + std::cout << " --log-file= file to log debug output\n"; + std::cout << " --debug-rbd-mirror=/ set rbd-mirror debug level\n"; + generic_server_usage(); +} + +static atomic_t g_stopping; + +static void handle_signal(int signum) +{ + g_stopping.set(1); +} + +int get_image_id(rbd::mirror::RadosRef cluster, int64_t pool_id, + const std::string &image_name, std::string *image_id) +{ + librados::IoCtx ioctx; + + int r = cluster->ioctx_create2(pool_id, ioctx); + if (r < 0) { + derr << "error opening ioctx for pool " << pool_id + << ": " << cpp_strerror(r) << dendl; + return r; + } + + librbd::ImageCtx *image_ctx = new librbd::ImageCtx(image_name, "", NULL, + ioctx, true); + r = image_ctx->state->open(); + if (r < 0) { + derr << "error opening remote image " << image_name + << ": " << cpp_strerror(r) << dendl; + delete image_ctx; + return r; + } + + *image_id = image_ctx->id; + image_ctx->state->close(); + return 0; +} + +int main(int argc, const char **argv) +{ + std::vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_DAEMON, + CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS); + + for (auto i = args.begin(); i != args.end(); ++i) { + if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { + usage(); + return EXIT_SUCCESS; + } + } + + if (args.size() < 3) { + usage(); + return EXIT_FAILURE; + } + + std::string local_pool_name = args[0]; + std::string remote_pool_name = args[1]; + std::string image_name = args[2]; + + dout(1) << "local_pool_name=" << local_pool_name << ", remote_pool_name=" + << remote_pool_name << ", image_name=" << image_name << dendl; + + rbd::mirror::ImageReplayer::BootstrapParams bootstap_params(local_pool_name, + image_name); + int64_t remote_pool_id; + std::string remote_image_id; + + if (local_pool_name == remote_pool_name) { + std::cerr << "local and remote pools can't be the same" << std::endl; + return EXIT_FAILURE; + } + + if (g_conf->daemonize) { + global_init_daemonize(g_ceph_context); + } + g_ceph_context->enable_perf_counter(); + + common_init_finish(g_ceph_context); + + init_async_signal_handler(); + register_async_signal_handler(SIGHUP, sighup_handler); + register_async_signal_handler_oneshot(SIGINT, handle_signal); + register_async_signal_handler_oneshot(SIGTERM, handle_signal); + + dout(5) << "connecting to cluster" << dendl; + + rbd::mirror::RadosRef local(new librados::Rados()); + rbd::mirror::RadosRef remote(new librados::Rados()); + + int r = local->init_with_context(g_ceph_context); + if (r < 0) { + derr << "could not initialize rados handle" << dendl; + goto cleanup; + } + + r = local->connect(); + if (r < 0) { + derr << "error connecting to local cluster" << dendl; + goto cleanup; + } + + r = remote->init_with_context(g_ceph_context); + if (r < 0) { + derr << "could not initialize rados handle" << dendl; + goto cleanup; + } + + r = remote->connect(); + if (r < 0) { + derr << "error connecting to local cluster" << dendl; + goto cleanup; + } + + r = remote->pool_lookup(remote_pool_name.c_str()); + if (r < 0) { + derr << "error finding remote pool " << remote_pool_name + << ": " << cpp_strerror(r) << dendl; + goto cleanup; + } + remote_pool_id = r; + + r = get_image_id(remote, remote_pool_id, image_name, &remote_image_id); + if (r < 0) { + derr << "error resolving ID for remote image " << image_name + << ": " << cpp_strerror(r) << dendl; + goto cleanup; + } + + dout(5) << "starting replay" << dendl; + + replayer = new rbd::mirror::ImageReplayer(local, remote, remote_pool_id, + remote_image_id); + + r = replayer->start(&bootstap_params); + if (r < 0) { + derr << "failed to start: " << cpp_strerror(r) << dendl; + goto cleanup; + } + + dout(5) << "replay started" << dendl; + + while (!g_stopping.read()) { + usleep(200000); + } + + dout(1) << "termination signal received, stopping replay" << dendl; + + replayer->stop(); + + dout(1) << "shutdown" << dendl; + + cleanup: + unregister_async_signal_handler(SIGHUP, sighup_handler); + unregister_async_signal_handler(SIGINT, handle_signal); + unregister_async_signal_handler(SIGTERM, handle_signal); + shutdown_async_signal_handler(); + + delete replayer; + g_ceph_context->put(); + + return r < 0 ? EXIT_SUCCESS : EXIT_FAILURE; +}