librbd: add options to enable balanced or localized reads for snapshots

Since snapshots never change, it's safe to read from replicas for them.
A common use for this would be reading from a parent snapshot shared by
many clones.

Convert LibrbdWriteback and AioRead to use the ObjectOperation api
so we can set flags. Fortunately the external wrapper holds no data,
so its lifecycle doesn't need to be managed.

Include a simple workunit that sets the flags in various combinations
and looks for their presence in the logs from 'rbd export'.

Fixes: #3064
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/qa/workunits/rbd/read-flags.sh b/qa/workunits/rbd/read-flags.sh
new file mode 100755 (executable)
index 0000000..375b3f3
--- /dev/null
+++ b/qa/workunits/rbd/read-flags.sh
@@ -0,0 +1,60 @@
+#!/bin/bash -ex
+
+# create a snapshot, then export it and check that setting read flags works
+# by looking at --debug-ms output
+
+function clean_up {
+    rm -f test.log || true
+    rbd snap remove test@snap || true
+    rbd rm test || true
+}
+
+function test_read_flags {
+    local IMAGE=$1
+    local SET_BALANCED=$2
+    local SET_LOCALIZED=$3
+    local EXPECT_BALANCED=$4
+    local EXPECT_LOCALIZED=$5
+
+    local EXTRA_ARGS="--log-file test.log --debug-ms 1 --no-log-to-stderr"
+    if [ "$SET_BALANCED" = 'y' ]; then
+       EXTRA_ARGS="$EXTRA_ARGS --rbd-balance-snap-reads"
+    elif [ "$SET_LOCALIZED" = 'y' ]; then
+       EXTRA_ARGS="$EXTRA_ARGS --rbd-localize-snap-reads"
+    fi
+
+    rbd export $IMAGE - $EXTRA_ARGS > /dev/null
+    if [ "$EXPECT_BALANCED" = 'y' ]; then
+       grep -q balance_reads test.log
+    else
+       grep -L balance_reads test.log | grep -q test.log
+    fi
+    if [ "$EXPECT_LOCALIZED" = 'y' ]; then
+       grep -q localize_reads test.log
+    else
+       grep -L localize_reads test.log | grep -q test.log
+    fi
+    rm -f test.log
+
+}
+
+clean_up
+
+trap clean_up INT TERM EXIT
+
+rbd create -s 10 test
+rbd snap create test@snap
+
+# export from non snapshot with or without settings should not have flags
+test_read_flags test n n n n
+test_read_flags test y y n n
+
+# export from snapshot should have read flags in log if they are set
+test_read_flags test@snap n n n n
+test_read_flags test@snap y n y n
+test_read_flags test@snap n y n y
+
+# balanced_reads happens to take priority over localize_reads
+test_read_flags test@snap y y y n
+
+echo OK

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 4510fb2cb8e73c4b8b6743b57900ec811034aad4..ffb62be568ff868a31b9f83140199d97fe2b4df0 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -543,6 +543,8 @@ OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 16<<20) // target dirty limit in by
 OPTION(rbd_cache_max_dirty_age, OPT_FLOAT, 1.0)      // seconds in cache before writeback starts
 OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writes to the cache before the aio_write call completes (true), or block before the aio completion is called (false)
 OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image
+OPTION(rbd_balance_snap_reads, OPT_BOOL, false)
+OPTION(rbd_localize_snap_reads, OPT_BOOL, false)
 
 OPTION(nss_db_path, OPT_STR, "") // path to nss db
 

diff --git a/src/librbd/AioRequest.cc b/src/librbd/AioRequest.cc
index 83e19272f8d2fbc44d8aa7f0caa10b5aaa5176be..11e111f75142be2fb5ff3a852811a65fa575f08a 100644 (file)
--- a/src/librbd/AioRequest.cc
+++ b/src/librbd/AioRequest.cc
@@ -91,15 +91,16 @@ namespace librbd {
     librados::AioCompletion *rados_completion =
       librados::Rados::aio_create_completion(this, rados_req_cb, NULL);
     int r;
+    librados::ObjectReadOperation op;
+    int flags = m_ictx->get_read_flags(m_snap_id);
     if (m_sparse) {
-      r = m_ioctx->aio_sparse_read(m_oid, rados_completion, &m_ext_map,
-                                  &m_read_data, m_object_len, m_object_off,
-                                  m_snap_id);
+      op.sparse_read(m_object_off, m_object_len, &m_ext_map, &m_read_data,
+                    NULL);
     } else {
-      r = m_ioctx->aio_read(m_oid, rados_completion, &m_read_data,
-                           m_object_len, m_object_off,
-                           m_snap_id);
+      op.read(m_object_off, m_object_len, &m_read_data, NULL);
     }
+    r = m_ioctx->aio_operate(m_oid, rados_completion, &op, m_snap_id, flags, NULL);
+
     rados_completion->release();
     return r;
   }
@@ -116,7 +117,8 @@ namespace librbd {
                               const ::SnapContext &snapc, librados::snap_t snap_id,
                               Context *completion,
                               bool hide_enoent)
-    : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion, hide_enoent),
+    : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion,
+                hide_enoent),
       m_state(LIBRBD_AIO_WRITE_FLAT), m_snap_seq(snapc.seq.val)
   {
     m_object_image_extents = objectx;

diff --git a/src/librbd/AioRequest.h b/src/librbd/AioRequest.h
index 8ad15a4264c5e6c276c56f74c5f060d64abe32bc..ac74a99dad3134d25e7a111204e0af5574fa5961 100644 (file)
--- a/src/librbd/AioRequest.h
+++ b/src/librbd/AioRequest.h
@@ -66,7 +66,8 @@ namespace librbd {
            vector<pair<uint64_t,uint64_t> >& be,
            librados::snap_t snap_id, bool sparse,
            Context *completion)
-      : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion, false),
+      : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion,
+                  false),
        m_buffer_extents(be),
        m_tried_parent(false), m_sparse(sparse) {
     }

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 08c72ef4a60ba67a433cd066e70a2495deb28efa..6796bc8e05eedb99020fcc55fcbb8a1cb771efa5 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -238,6 +238,18 @@ namespace librbd {
     delete perfcounter;
   }
 
+  int ImageCtx::get_read_flags(snap_t snap_id) {
+    int flags = librados::OPERATION_NOFLAG;
+    if (snap_id == LIBRADOS_SNAP_HEAD)
+      return flags;
+
+    if (cct->_conf->rbd_balance_snap_reads)
+      flags |= librados::OPERATION_BALANCE_READS;
+    else if (cct->_conf->rbd_localize_snap_reads)
+      flags |= librados::OPERATION_LOCALIZE_READS;
+    return flags;
+  }
+
   int ImageCtx::snap_set(string in_snap_name)
   {
     map<string, SnapInfo>::iterator it = snaps_by_name.find(in_snap_name);

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index fde24fb3e247543d19462234c1f5fefc4b9e8719..29ca2f197eae1f479ffb4e1f1f6fc7595f756ad9 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -99,6 +99,7 @@ namespace librbd {
     void init_layout();
     void perf_start(std::string name);
     void perf_stop();
+    int get_read_flags(librados::snap_t snap_id);
     int snap_set(std::string in_snap_name);
     void snap_unset();
     librados::snap_t get_snap_id(std::string in_snap_name) const;

diff --git a/src/librbd/LibrbdWriteback.cc b/src/librbd/LibrbdWriteback.cc
index 237901dc61fed3303b74bb0428bc935ec94cd426..a7ab0488dc6e1e983633b17d72eaa05c52a9d6a1 100644 (file)
--- a/src/librbd/LibrbdWriteback.cc
+++ b/src/librbd/LibrbdWriteback.cc
@@ -100,8 +100,11 @@ namespace librbd {
     Context *req = new C_Request(m_ictx->cct, onfinish, &m_lock);
     librados::AioCompletion *rados_completion =
       librados::Rados::aio_create_completion(req, context_cb, NULL);
-    int r = m_ictx->data_ctx.aio_read(oid.name, rados_completion, pbl,
-                                     len, off);
+    librados::ObjectReadOperation op;
+    op.read(off, len, pbl, NULL);
+    int flags = m_ictx->get_read_flags(snapid);
+    int r = m_ictx->data_ctx.aio_operate(oid.name, rados_completion, &op,
+                                        snapid, flags, NULL);
     rados_completion->release();
     assert(r >= 0);
   }