From: Josh Durgin Date: Sun, 12 May 2013 21:53:26 +0000 (-0700) Subject: librbd: add options to enable balanced or localized reads for snapshots X-Git-Tag: v0.63~38^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=13ae13a9068afcd4eb4b3574c46875cad8c91ab6;p=ceph.git librbd: add options to enable balanced or localized reads for snapshots Since snapshots never change, it's safe to read from replicas for them. A common use for this would be reading from a parent snapshot shared by many clones. Convert LibrbdWriteback and AioRead to use the ObjectOperation api so we can set flags. Fortunately the external wrapper holds no data, so its lifecycle doesn't need to be managed. Include a simple workunit that sets the flags in various combinations and looks for their presence in the logs from 'rbd export'. Fixes: #3064 Signed-off-by: Josh Durgin --- diff --git a/qa/workunits/rbd/read-flags.sh b/qa/workunits/rbd/read-flags.sh new file mode 100755 index 00000000000..375b3f3ee15 --- /dev/null +++ b/qa/workunits/rbd/read-flags.sh @@ -0,0 +1,60 @@ +#!/bin/bash -ex + +# create a snapshot, then export it and check that setting read flags works +# by looking at --debug-ms output + +function clean_up { + rm -f test.log || true + rbd snap remove test@snap || true + rbd rm test || true +} + +function test_read_flags { + local IMAGE=$1 + local SET_BALANCED=$2 + local SET_LOCALIZED=$3 + local EXPECT_BALANCED=$4 + local EXPECT_LOCALIZED=$5 + + local EXTRA_ARGS="--log-file test.log --debug-ms 1 --no-log-to-stderr" + if [ "$SET_BALANCED" = 'y' ]; then + EXTRA_ARGS="$EXTRA_ARGS --rbd-balance-snap-reads" + elif [ "$SET_LOCALIZED" = 'y' ]; then + EXTRA_ARGS="$EXTRA_ARGS --rbd-localize-snap-reads" + fi + + rbd export $IMAGE - $EXTRA_ARGS > /dev/null + if [ "$EXPECT_BALANCED" = 'y' ]; then + grep -q balance_reads test.log + else + grep -L balance_reads test.log | grep -q test.log + fi + if [ "$EXPECT_LOCALIZED" = 'y' ]; then + grep -q localize_reads test.log + else + grep -L localize_reads test.log | grep -q test.log + fi + rm -f test.log + +} + +clean_up + +trap clean_up INT TERM EXIT + +rbd create -s 10 test +rbd snap create test@snap + +# export from non snapshot with or without settings should not have flags +test_read_flags test n n n n +test_read_flags test y y n n + +# export from snapshot should have read flags in log if they are set +test_read_flags test@snap n n n n +test_read_flags test@snap y n y n +test_read_flags test@snap n y n y + +# balanced_reads happens to take priority over localize_reads +test_read_flags test@snap y y y n + +echo OK diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 4510fb2cb8e..ffb62be568f 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -543,6 +543,8 @@ OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 16<<20) // target dirty limit in by OPTION(rbd_cache_max_dirty_age, OPT_FLOAT, 1.0) // seconds in cache before writeback starts OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writes to the cache before the aio_write call completes (true), or block before the aio completion is called (false) OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image +OPTION(rbd_balance_snap_reads, OPT_BOOL, false) +OPTION(rbd_localize_snap_reads, OPT_BOOL, false) OPTION(nss_db_path, OPT_STR, "") // path to nss db diff --git a/src/librbd/AioRequest.cc b/src/librbd/AioRequest.cc index 83e19272f8d..11e111f7514 100644 --- a/src/librbd/AioRequest.cc +++ b/src/librbd/AioRequest.cc @@ -91,15 +91,16 @@ namespace librbd { librados::AioCompletion *rados_completion = librados::Rados::aio_create_completion(this, rados_req_cb, NULL); int r; + librados::ObjectReadOperation op; + int flags = m_ictx->get_read_flags(m_snap_id); if (m_sparse) { - r = m_ioctx->aio_sparse_read(m_oid, rados_completion, &m_ext_map, - &m_read_data, m_object_len, m_object_off, - m_snap_id); + op.sparse_read(m_object_off, m_object_len, &m_ext_map, &m_read_data, + NULL); } else { - r = m_ioctx->aio_read(m_oid, rados_completion, &m_read_data, - m_object_len, m_object_off, - m_snap_id); + op.read(m_object_off, m_object_len, &m_read_data, NULL); } + r = m_ioctx->aio_operate(m_oid, rados_completion, &op, m_snap_id, flags, NULL); + rados_completion->release(); return r; } @@ -116,7 +117,8 @@ namespace librbd { const ::SnapContext &snapc, librados::snap_t snap_id, Context *completion, bool hide_enoent) - : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion, hide_enoent), + : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion, + hide_enoent), m_state(LIBRBD_AIO_WRITE_FLAT), m_snap_seq(snapc.seq.val) { m_object_image_extents = objectx; diff --git a/src/librbd/AioRequest.h b/src/librbd/AioRequest.h index 8ad15a4264c..ac74a99dad3 100644 --- a/src/librbd/AioRequest.h +++ b/src/librbd/AioRequest.h @@ -66,7 +66,8 @@ namespace librbd { vector >& be, librados::snap_t snap_id, bool sparse, Context *completion) - : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion, false), + : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion, + false), m_buffer_extents(be), m_tried_parent(false), m_sparse(sparse) { } diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index 08c72ef4a60..6796bc8e05e 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -238,6 +238,18 @@ namespace librbd { delete perfcounter; } + int ImageCtx::get_read_flags(snap_t snap_id) { + int flags = librados::OPERATION_NOFLAG; + if (snap_id == LIBRADOS_SNAP_HEAD) + return flags; + + if (cct->_conf->rbd_balance_snap_reads) + flags |= librados::OPERATION_BALANCE_READS; + else if (cct->_conf->rbd_localize_snap_reads) + flags |= librados::OPERATION_LOCALIZE_READS; + return flags; + } + int ImageCtx::snap_set(string in_snap_name) { map::iterator it = snaps_by_name.find(in_snap_name); diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index fde24fb3e24..29ca2f197ea 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -99,6 +99,7 @@ namespace librbd { void init_layout(); void perf_start(std::string name); void perf_stop(); + int get_read_flags(librados::snap_t snap_id); int snap_set(std::string in_snap_name); void snap_unset(); librados::snap_t get_snap_id(std::string in_snap_name) const; diff --git a/src/librbd/LibrbdWriteback.cc b/src/librbd/LibrbdWriteback.cc index 237901dc61f..a7ab0488dc6 100644 --- a/src/librbd/LibrbdWriteback.cc +++ b/src/librbd/LibrbdWriteback.cc @@ -100,8 +100,11 @@ namespace librbd { Context *req = new C_Request(m_ictx->cct, onfinish, &m_lock); librados::AioCompletion *rados_completion = librados::Rados::aio_create_completion(req, context_cb, NULL); - int r = m_ictx->data_ctx.aio_read(oid.name, rados_completion, pbl, - len, off); + librados::ObjectReadOperation op; + op.read(off, len, pbl, NULL); + int flags = m_ictx->get_read_flags(snapid); + int r = m_ictx->data_ctx.aio_operate(oid.name, rados_completion, &op, + snapid, flags, NULL); rados_completion->release(); assert(r >= 0); }