rbd: use IO hints for read/write operations

Import, export, and benchmark operations now use the librados
IO hints to indicate the read/write pattern and whether caching
is required.

Fixes: #10462
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
Reviewed-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/src/rbd.cc b/src/rbd.cc
index 866906c900bee14a4f6868a684a1b6786cb4d0e7..bc7dcbd96e61158305c5cd9b42de4390142f1d1f 100644 (file)
--- a/src/rbd.cc
+++ b/src/rbd.cc
@@ -35,6 +35,7 @@
 #include <boost/scope_exit.hpp>
 #include <boost/scoped_ptr.hpp>
 #include <errno.h>
+#include <fcntl.h>
 #include <iostream>
 #include <memory>
 #include <sstream>
@@ -863,7 +864,8 @@ struct rbd_bencher {
       in_flight(0)
   { }
 
-  bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl)
+  bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl,
+                  int op_flags)
   {
     {
       Mutex::Locker l(lock);
@@ -873,7 +875,7 @@ struct rbd_bencher {
     }
     librbd::RBD::AioCompletion *c =
       new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion);
-    image->aio_write(off, len, bl, c);
+    image->aio_write2(off, len, bl, c, op_flags);
     //cout << "start " << c << " at " << off << "~" << len << std::endl;
     return true;
   }
@@ -960,13 +962,20 @@ static int do_bench_write(librbd::Image& image, uint64_t io_size,
   uint64_t cur_ios = 0;
   uint64_t cur_off = 0;
 
+  int op_flags;
+  if  (pattern == "rand") {
+    op_flags = LIBRADOS_OP_FLAG_FADVISE_RANDOM;
+  } else {
+    op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL;
+  }
+
   printf("  SEC       OPS   OPS/SEC   BYTES/SEC\n");
   uint64_t off;
   for (off = 0; off < io_bytes; ) {
     b.wait_for(io_threads - 1);
     i = 0;
     while (i < io_threads && off < io_bytes &&
-          b.start_write(io_threads, thread_offset[i], io_size, bl)) {
+          b.start_write(io_threads, thread_offset[i], io_size, bl, op_flags)) {
       ++i;
       ++ios;
       off += io_size;
@@ -1044,7 +1053,11 @@ public:
       m_fd(fd)
   {
     m_throttle.start_op();
-    int r = image.aio_read(offset, length, m_bufferlist, m_aio_completion);
+
+    int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
+                  LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
+    int r = image.aio_read2(offset, length, m_bufferlist, m_aio_completion,
+                           op_flags);
     if (r < 0) {
       cerr << "rbd: error requesting read from source image" << std::endl;
       m_throttle.end_op(r);
@@ -1126,6 +1139,7 @@ static int do_export(librbd::Image& image, const char *path)
     if (fd < 0) {
       return -errno;
     }
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
   }
 
   MyProgressContext pc("Exporting image");
@@ -1173,7 +1187,7 @@ static int export_diff_cb(uint64_t ofs, size_t _len, int exists, void *arg)
   if (exists) {
     // read block
     bl.clear();
-    r = ec->image->read(ofs, len, bl);
+    r = ec->image->read2(ofs, len, bl, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
     if (r < 0)
       return r;
     r = bl.write_fd(ec->fd);
@@ -1383,8 +1397,10 @@ public:
   {
     m_throttle.start_op();
 
-    int r = image.aio_write(m_offset, m_bufferlist.length(), m_bufferlist,
-                           m_aio_completion);
+    int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
+                  LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
+    int r = image.aio_write2(m_offset, m_bufferlist.length(), m_bufferlist,
+                            m_aio_completion, op_flags);
     if (r < 0) {
       cerr << "rbd: error requesting write to destination image" << std::endl;
       m_throttle.end_op(r);
@@ -1481,6 +1497,8 @@ static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx,
       assert(bdev_size >= 0);
       size = (uint64_t) bdev_size;
     }
+
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
   }
   r = do_create(rbd, io_ctx, imgname, size, order, format, features, 0, 0);
   if (r < 0) {
@@ -1692,7 +1710,7 @@ static int do_import_diff(librbd::Image &image, const char *path)
        bufferlist data;
        data.append(bp);
        dout(2) << " write " << off << "~" << len << dendl;
-       image.write(off, len, data);
+       image.write2(off, len, data, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
       } else {
        dout(2) << " zero " << off << "~" << len << dendl;
        image.discard(off, len);