From ea7b10501c02fd0df3ff1d548185ac14082344e4 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Sep 2015 12:38:33 -0400 Subject: [PATCH] librbd: new rbd_aio_readv/rbd_aio_writev API methods The RBD C API now has scatter/gather IO support via the API. The C++ API uses bufferlists which can also support scatter/gather IO. Fixes: http://tracker.ceph.com/issues/13025 Signed-off-by: Jason Dillaman --- src/include/rbd/librbd.h | 19 ++++++--- src/librbd/CMakeLists.txt | 2 +- src/librbd/librbd.cc | 72 +++++++++++++++++++++++++++++++++ src/test/librbd/test_librbd.cc | 73 ++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 7 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 1bdc3551e9fd5..c003c414b6e2e 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -27,22 +27,24 @@ extern "C" { #endif #include #include +#include #include "../rados/librados.h" #include "features.h" -#define LIBRBD_VER_MAJOR 0 -#define LIBRBD_VER_MINOR 1 -#define LIBRBD_VER_EXTRA 11 +#define LIBRBD_VER_MAJOR 1 +#define LIBRBD_VER_MINOR 12 +#define LIBRBD_VER_EXTRA 0 #define LIBRBD_VERSION(maj, min, extra) ((maj << 16) + (min << 8) + extra) #define LIBRBD_VERSION_CODE LIBRBD_VERSION(LIBRBD_VER_MAJOR, LIBRBD_VER_MINOR, LIBRBD_VER_EXTRA) -#define LIBRBD_SUPPORTS_WATCH 0 #define LIBRBD_SUPPORTS_AIO_FLUSH 1 -#define LIBRBD_SUPPORTS_INVALIDATE 1 #define LIBRBD_SUPPORTS_AIO_OPEN 1 #define LIBRBD_SUPPORTS_LOCKING 1 +#define LIBRBD_SUPPORTS_INVALIDATE 1 +#define LIBRBD_SUPPORTS_IOVEC 1 +#define LIBRBD_SUPPORTS_WATCH 0 #if __GNUC__ >= 4 #define CEPH_RBD_API __attribute__ ((visibility ("default"))) @@ -643,7 +645,10 @@ CEPH_RBD_API int rbd_aio_write(rbd_image_t image, uint64_t off, size_t len, * @param op_flags: see librados.h constants beginning with LIBRADOS_OP_FLAG */ CEPH_RBD_API int rbd_aio_write2(rbd_image_t image, uint64_t off, size_t len, - const char *buf, rbd_completion_t c, int op_flags); + const char *buf, rbd_completion_t c, + int op_flags); +CEPH_RBD_API int rbd_aio_writev(rbd_image_t image, const struct iovec *iov, + int iovcnt, uint64_t off, rbd_completion_t c); CEPH_RBD_API int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len, char *buf, rbd_completion_t c); /* @@ -651,6 +656,8 @@ CEPH_RBD_API int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len, */ CEPH_RBD_API int rbd_aio_read2(rbd_image_t image, uint64_t off, size_t len, char *buf, rbd_completion_t c, int op_flags); +CEPH_RBD_API int rbd_aio_readv(rbd_image_t image, const struct iovec *iov, + int iovcnt, uint64_t off, rbd_completion_t c); CEPH_RBD_API int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len, rbd_completion_t c); diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index af0fdcc8deb68..cfd0951aefc65 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -125,7 +125,7 @@ endif() if(ENABLE_SHARED) set_target_properties(librbd PROPERTIES OUTPUT_NAME rbd - VERSION 1.0.0 + VERSION 1.12.0 SOVERSION 1 COMPILE_FLAGS "-fvisibility=hidden -fvisibility-inlines-hidden" LINK_FLAGS "-Wl,--exclude-libs,ALL") diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 35bbcf7db71e0..4b8cb47fe846f 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -2943,6 +2943,41 @@ extern "C" int rbd_aio_write2(rbd_image_t image, uint64_t off, size_t len, return 0; } +extern "C" int rbd_aio_writev(rbd_image_t image, const struct iovec *iov, + int iovcnt, uint64_t off, rbd_completion_t c) +{ + librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + + // convert the scatter list into a bufferlist + ssize_t len = 0; + bufferlist bl; + for (int i = 0; i < iovcnt; ++i) { + const struct iovec &io = iov[i]; + len += io.iov_len; + if (len < 0) { + break; + } + + bl.push_back(create_write_raw(ictx, static_cast(io.iov_base), + io.iov_len)); + } + + int r = 0; + if (iovcnt <= 0 || len < 0) { + r = -EINVAL; + } + + tracepoint(librbd, aio_write_enter, ictx, ictx->name.c_str(), + ictx->snap_name.c_str(), ictx->read_only, off, len, NULL, + comp->pc); + if (r == 0) { + ictx->io_work_queue->aio_write(get_aio_completion(comp), off, len, + std::move(bl), 0); + } + tracepoint(librbd, aio_write_exit, r); + return r; +} extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len, rbd_completion_t c) @@ -2980,6 +3015,43 @@ extern "C" int rbd_aio_read2(rbd_image_t image, uint64_t off, size_t len, return 0; } +extern "C" int rbd_aio_readv(rbd_image_t image, const struct iovec *iov, + int iovcnt, uint64_t off, rbd_completion_t c) +{ + librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; + librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c; + + ssize_t len = 0; + for (int i = 0; i < iovcnt; ++i) { + len += iov[i].iov_len; + if (len < 0) { + break; + } + } + + int r = 0; + if (iovcnt == 0 || len < 0) { + r = -EINVAL; + } + + tracepoint(librbd, aio_read_enter, ictx, ictx->name.c_str(), + ictx->snap_name.c_str(), ictx->read_only, off, len, NULL, + comp->pc); + if (r == 0) { + librbd::io::ReadResult read_result; + if (iovcnt == 1) { + read_result = std::move(librbd::io::ReadResult( + static_cast(iov[0].iov_base), iov[0].iov_len)); + } else { + read_result = std::move(librbd::io::ReadResult(iov, iovcnt)); + } + ictx->io_work_queue->aio_read(get_aio_completion(comp), off, len, + std::move(read_result), 0); + } + tracepoint(librbd, aio_read_exit, r); + return r; +} + extern "C" int rbd_flush(rbd_image_t image) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index 89d0942ab866c..320170b3a59cd 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -1622,6 +1622,79 @@ TEST_F(TestLibRBD, TestDataPoolIO) rados_ioctx_destroy(ioctx); } + +TEST_F(TestLibRBD, TestScatterGatherIO) +{ + rados_ioctx_t ioctx; + rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx); + + rbd_image_t image; + int order = 0; + std::string name = get_temp_image_name(); + uint64_t size = 20 << 20; + + ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order)); + ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL)); + + std::string write_buffer("This is a test"); + struct iovec bad_iovs[] = { + {.iov_base = NULL, .iov_len = static_cast(-1)} + }; + struct iovec write_iovs[] = { + {.iov_base = &write_buffer[0], .iov_len = 5}, + {.iov_base = &write_buffer[5], .iov_len = 3}, + {.iov_base = &write_buffer[8], .iov_len = 2}, + {.iov_base = &write_buffer[10], .iov_len = 4} + }; + + rbd_completion_t comp; + rbd_aio_create_completion(NULL, NULL, &comp); + ASSERT_EQ(-EINVAL, rbd_aio_writev(image, write_iovs, 0, 0, comp)); + ASSERT_EQ(-EINVAL, rbd_aio_writev(image, bad_iovs, 1, 0, comp)); + ASSERT_EQ(0, rbd_aio_writev(image, write_iovs, + sizeof(write_iovs) / sizeof(struct iovec), + 1<