librbd: add internal support for scatter/gather IO

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/AioImageRequest.cc b/src/librbd/AioImageRequest.cc
index 8c6b632d0e2763d055f658f561afb68db5f5fa55..4604741feb9906b8141786fa330954fa04d923ba 100644 (file)
--- a/src/librbd/AioImageRequest.cc
+++ b/src/librbd/AioImageRequest.cc
@@ -150,17 +150,9 @@ private:
 
 template <typename I>
 void AioImageRequest<I>::aio_read(I *ictx, AioCompletion *c,
-                                  const Extents &extents, char *buf,
+                                  Extents &&image_extents, char *buf,
                                   bufferlist *pbl, int op_flags) {
-  AioImageRead<I> req(*ictx, c, extents, buf, pbl, op_flags);
-  req.send();
-}
-
-template <typename I>
-void AioImageRequest<I>::aio_read(I *ictx, AioCompletion *c, uint64_t off,
-                                 size_t len, char *buf, bufferlist *pbl,
-                                 int op_flags) {
-  AioImageRead<I> req(*ictx, c, off, len, buf, pbl, op_flags);
+  AioImageRead<I> req(*ictx, c, std::move(image_extents), buf, pbl, op_flags);
   req.send();
 }
 
@@ -172,9 +164,11 @@ void AioImageRequest<I>::aio_write(I *ictx, AioCompletion *c, uint64_t off,
 }
 
 template <typename I>
-void AioImageRequest<I>::aio_write(I *ictx, AioCompletion *c, uint64_t off,
-                                   bufferlist &&bl, int op_flags) {
-  AioImageWrite<I> req(*ictx, c, off, std::move(bl), op_flags);
+void AioImageRequest<I>::aio_write(I *ictx, AioCompletion *c,
+                                   Extents &&image_extents, bufferlist &&bl,
+                                   int op_flags) {
+  AioImageWrite<I> req(*ictx, c, std::move(image_extents), std::move(bl),
+                       op_flags);
   req.send();
 }
 
@@ -218,9 +212,10 @@ void AioImageRead<I>::send_request() {
   I &image_ctx = this->m_image_ctx;
   CephContext *cct = image_ctx.cct;
 
+  auto &image_extents = this->m_image_extents;
   if (image_ctx.object_cacher && image_ctx.readahead_max_bytes > 0 &&
       !(m_op_flags & LIBRADOS_OP_FLAG_FADVISE_RANDOM)) {
-    readahead(get_image_ctx(&image_ctx), m_image_extents);
+    readahead(get_image_ctx(&image_ctx), image_extents);
   }
 
   AioCompletion *aio_comp = this->m_aio_comp;
@@ -234,11 +229,9 @@ void AioImageRead<I>::send_request() {
     snap_id = image_ctx.snap_id;
 
     // map
-    for (vector<pair<uint64_t,uint64_t> >::const_iterator p =
-           m_image_extents.begin();
-         p != m_image_extents.end(); ++p) {
-      uint64_t len = p->second;
-      int r = clip_io(get_image_ctx(&image_ctx), p->first, &len);
+    for (auto &extent : image_extents) {
+      uint64_t len = extent.second;
+      int r = clip_io(get_image_ctx(&image_ctx), extent.first, &len);
       if (r < 0) {
         aio_comp->fail(r);
         return;
@@ -248,7 +241,7 @@ void AioImageRead<I>::send_request() {
       }
 
       Striper::file_to_extents(cct, image_ctx.format_string,
-                               &image_ctx.layout, p->first, len, 0,
+                               &image_ctx.layout, extent.first, len, 0,
                                object_extents, buffer_ofs);
       buffer_ofs += len;
     }
@@ -306,7 +299,7 @@ void AbstractAioImageWrite<I>::send_request() {
   bool journaling = false;
 
   AioCompletion *aio_comp = this->m_aio_comp;
-  uint64_t clip_len = m_len;
+  uint64_t clip_len = 0;
   ObjectExtents object_extents;
   ::SnapContext snapc;
   {
@@ -318,21 +311,25 @@ void AbstractAioImageWrite<I>::send_request() {
       return;
     }
 
-    int r = clip_io(get_image_ctx(&image_ctx), m_off, &clip_len);
-    if (r < 0) {
-      aio_comp->fail(r);
-      return;
-    }
-
-    snapc = image_ctx.snapc;
+    for (auto &extent : this->m_image_extents) {
+      uint64_t len = extent.second;
+      int r = clip_io(get_image_ctx(&image_ctx), extent.first, &len);
+      if (r < 0) {
+        aio_comp->fail(r);
+        return;
+      }
+      if (len == 0) {
+        continue;
+      }
 
-    // map to object extents
-    if (clip_len > 0) {
+      // map to object extents
       Striper::file_to_extents(cct, image_ctx.format_string,
-                               &image_ctx.layout, m_off, clip_len, 0,
+                               &image_ctx.layout, extent.first, len, 0,
                                object_extents);
+      clip_len += len;
     }
 
+    snapc = image_ctx.snapc;
     journaling = (image_ctx.journal != nullptr &&
                   image_ctx.journal->is_journal_appending());
   }
@@ -408,9 +405,19 @@ template <typename I>
 uint64_t AioImageWrite<I>::append_journal_event(
     const AioObjectRequests &requests, bool synchronous) {
   I &image_ctx = this->m_image_ctx;
-  uint64_t tid = image_ctx.journal->append_write_event(this->m_off, this->m_len,
-                                                       m_bl, requests,
-                                                       synchronous);
+
+  uint64_t tid;
+  uint64_t buffer_offset = 0;
+  assert(!this->m_image_extents.empty());
+  for (auto &extent : this->m_image_extents) {
+    bufferlist sub_bl;
+    sub_bl.substr_of(m_bl, buffer_offset, extent.second);
+    buffer_offset += extent.second;
+
+    tid = image_ctx.journal->append_write_event(extent.first, extent.second,
+                                                sub_bl, requests, synchronous);
+  }
+
   if (image_ctx.object_cacher == NULL) {
     AioCompletion *aio_comp = this->m_aio_comp;
     aio_comp->associate_journal_event(tid);
@@ -476,11 +483,15 @@ uint64_t AioImageDiscard<I>::append_journal_event(
     const AioObjectRequests &requests, bool synchronous) {
   I &image_ctx = this->m_image_ctx;
 
-  journal::EventEntry event_entry(journal::AioDiscardEvent(this->m_off,
-                                                           this->m_len));
-  uint64_t tid = image_ctx.journal->append_io_event(std::move(event_entry),
-                                                    requests, this->m_off,
-                                                    this->m_len, synchronous);
+  uint64_t tid;
+  assert(!this->m_image_extents.empty());
+  for (auto &extent : this->m_image_extents) {
+    journal::EventEntry event_entry(journal::AioDiscardEvent(extent.first,
+                                                             extent.second));
+    tid = image_ctx.journal->append_io_event(std::move(event_entry),
+                                             requests, extent.first,
+                                             extent.second, synchronous);
+  }
 
   AioCompletion *aio_comp = this->m_aio_comp;
   aio_comp->associate_journal_event(tid);

diff --git a/src/librbd/AioImageRequest.h b/src/librbd/AioImageRequest.h
index 238989ed4e3404f8bac42a10fce57d7f5414433c..3d57cd408a85a0cec80929686e8c1a5cdd5bdcd9 100644 (file)
--- a/src/librbd/AioImageRequest.h
+++ b/src/librbd/AioImageRequest.h
@@ -27,14 +27,12 @@ public:
   virtual ~AioImageRequest() {}
 
   static void aio_read(ImageCtxT *ictx, AioCompletion *c,
-                       const std::vector<std::pair<uint64_t,uint64_t> > &extents,
-                       char *buf, bufferlist *pbl, int op_flags);
-  static void aio_read(ImageCtxT *ictx, AioCompletion *c, uint64_t off,
-                       size_t len, char *buf, bufferlist *pbl, int op_flags);
+                       Extents &&image_extents, char *buf, bufferlist *pbl,
+                       int op_flags);
   static void aio_write(ImageCtxT *ictx, AioCompletion *c, uint64_t off,
                         size_t len, const char *buf, int op_flags);
-  static void aio_write(ImageCtxT *ictx, AioCompletion *c, uint64_t off,
-                        bufferlist &&bl, int op_flags);
+  static void aio_write(ImageCtxT *ictx, AioCompletion *c,
+                        Extents &&image_extents, bufferlist &&bl, int op_flags);
   static void aio_discard(ImageCtxT *ictx, AioCompletion *c, uint64_t off,
                           uint64_t len);
   static void aio_flush(ImageCtxT *ictx, AioCompletion *c);
@@ -55,9 +53,13 @@ protected:
 
   ImageCtxT &m_image_ctx;
   AioCompletion *m_aio_comp;
+  Extents m_image_extents;
 
-  AioImageRequest(ImageCtxT &image_ctx, AioCompletion *aio_comp)
-    : m_image_ctx(image_ctx), m_aio_comp(aio_comp) {}
+  AioImageRequest(ImageCtxT &image_ctx, AioCompletion *aio_comp,
+                  Extents &&image_extents)
+    : m_image_ctx(image_ctx), m_aio_comp(aio_comp),
+      m_image_extents(image_extents) {
+  }
 
   virtual void send_request() = 0;
   virtual aio_type_t get_aio_type() const = 0;
@@ -69,19 +71,11 @@ class AioImageRead : public AioImageRequest<ImageCtxT> {
 public:
   using typename AioImageRequest<ImageCtxT>::Extents;
 
-  AioImageRead(ImageCtxT &image_ctx, AioCompletion *aio_comp, uint64_t off,
-               size_t len, char *buf, bufferlist *pbl, int op_flags)
-    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp), m_buf(buf), m_pbl(pbl),
-      m_op_flags(op_flags) {
-    m_image_extents.push_back(std::make_pair(off, len));
-  }
-
   AioImageRead(ImageCtxT &image_ctx, AioCompletion *aio_comp,
-               const Extents &image_extents, char *buf, bufferlist *pbl,
+               Extents &&image_extents, char *buf, bufferlist *pbl,
                int op_flags)
-    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp),
-      m_image_extents(image_extents), m_buf(buf), m_pbl(pbl),
-      m_op_flags(op_flags) {
+    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp, std::move(image_extents)),
+      m_buf(buf), m_pbl(pbl), m_op_flags(op_flags) {
   }
 
 protected:
@@ -93,7 +87,6 @@ protected:
     return "aio_read";
   }
 private:
-  Extents m_image_extents;
   char *m_buf;
   bufferlist *m_pbl;
   int m_op_flags;
@@ -112,15 +105,13 @@ public:
 
 protected:
   using typename AioImageRequest<ImageCtxT>::AioObjectRequests;
+  using typename AioImageRequest<ImageCtxT>::Extents;
 
   typedef std::vector<ObjectExtent> ObjectExtents;
 
-  const uint64_t m_off;
-  const size_t m_len;
-
   AbstractAioImageWrite(ImageCtxT &image_ctx, AioCompletion *aio_comp,
-                        uint64_t off, size_t len)
-    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp), m_off(off), m_len(len),
+                        Extents &&image_extents)
+    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp, std::move(image_extents)),
       m_synchronous(false) {
   }
 
@@ -152,15 +143,18 @@ private:
 template <typename ImageCtxT = ImageCtx>
 class AioImageWrite : public AbstractAioImageWrite<ImageCtxT> {
 public:
+  using typename AioImageRequest<ImageCtxT>::Extents;
+
   AioImageWrite(ImageCtxT &image_ctx, AioCompletion *aio_comp, uint64_t off,
                 size_t len, const char *buf, int op_flags)
-    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp, off, len),
+    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp, {{off, len}}),
       m_op_flags(op_flags) {
     m_bl.append(buf, len);
   }
-  AioImageWrite(ImageCtxT &image_ctx, AioCompletion *aio_comp, uint64_t off,
-                bufferlist &&bl, int op_flags)
-    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp, off, bl.length()),
+  AioImageWrite(ImageCtxT &image_ctx, AioCompletion *aio_comp,
+                Extents &&image_extents, bufferlist &&bl, int op_flags)
+    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp,
+                                       std::move(image_extents)),
       m_bl(std::move(bl)), m_op_flags(op_flags) {
   }
 
@@ -200,7 +194,7 @@ class AioImageDiscard : public AbstractAioImageWrite<ImageCtxT> {
 public:
   AioImageDiscard(ImageCtxT &image_ctx, AioCompletion *aio_comp, uint64_t off,
                   uint64_t len)
-    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp, off, len) {
+    : AbstractAioImageWrite<ImageCtxT>(image_ctx, aio_comp, {{off, len}}) {
   }
 
 protected:
@@ -232,7 +226,7 @@ template <typename ImageCtxT = ImageCtx>
 class AioImageFlush : public AioImageRequest<ImageCtxT> {
 public:
   AioImageFlush(ImageCtxT &image_ctx, AioCompletion *aio_comp)
-    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp) {
+    : AioImageRequest<ImageCtxT>(image_ctx, aio_comp, {}) {
   }
 
   virtual bool is_write_op() const {

diff --git a/src/librbd/AioImageRequestWQ.cc b/src/librbd/AioImageRequestWQ.cc
index 08cdadb244001dad24c133ee0e72658cb9d98cfd..8cbc5a05fbd13b59b502232b5db6028bd8fc6293 100644 (file)
--- a/src/librbd/AioImageRequestWQ.cc
+++ b/src/librbd/AioImageRequestWQ.cc
@@ -120,10 +120,11 @@ void AioImageRequestWQ::aio_read(AioCompletion *c, uint64_t off, uint64_t len,
 
   if (m_image_ctx.non_blocking_aio || writes_blocked() || !writes_empty() ||
       lock_required) {
-    queue(new AioImageRead<>(m_image_ctx, c, off, len, buf, pbl, op_flags));
+    queue(new AioImageRead<>(m_image_ctx, c, {{off, len}}, buf, pbl, op_flags));
   } else {
     c->start_op();
-    AioImageRequest<>::aio_read(&m_image_ctx, c, off, len, buf, pbl, op_flags);
+    AioImageRequest<>::aio_read(&m_image_ctx, c, {{off, len}}, buf, pbl,
+                                op_flags);
     finish_in_flight_op();
   }
 }

diff --git a/src/librbd/AioObjectRequest.cc b/src/librbd/AioObjectRequest.cc
index 5a5f1e074220388fb94876f9549008993bce2fb7..6226a0d3470296ca596a1d6dcd094440ae48f287 100644 (file)
--- a/src/librbd/AioObjectRequest.cc
+++ b/src/librbd/AioObjectRequest.cc
@@ -212,7 +212,7 @@ bool AioObjectRead<I>::should_complete(int r)
             m_state = LIBRBD_AIO_READ_COPYUP;
           }
 
-          read_from_parent(parent_extents);
+          read_from_parent(std::move(parent_extents));
           finished = false;
         }
       }
@@ -303,16 +303,18 @@ void AioObjectRead<I>::send_copyup()
     image_ctx->copyup_list.find(this->m_object_no);
   if (it == image_ctx->copyup_list.end()) {
     // create and kick off a CopyupRequest
-    CopyupRequest *new_req = new CopyupRequest(image_ctx, this->m_oid,
-                                               this->m_object_no,
-                                               this->m_parent_extents);
+    CopyupRequest *new_req = new CopyupRequest(
+      image_ctx, this->m_oid, this->m_object_no,
+      std::move(this->m_parent_extents));
+    this->m_parent_extents.clear();
+
     image_ctx->copyup_list[this->m_object_no] = new_req;
     new_req->send();
   }
 }
 
 template <typename I>
-void AioObjectRead<I>::read_from_parent(const Extents& parent_extents)
+void AioObjectRead<I>::read_from_parent(Extents&& parent_extents)
 {
   ImageCtx *image_ctx = this->m_ictx;
   AioCompletion *parent_completion = AioCompletion::create_and_start<
@@ -323,7 +325,8 @@ void AioObjectRead<I>::read_from_parent(const Extents& parent_extents)
                             << " extents " << parent_extents
                             << dendl;
   AioImageRequest<>::aio_read(image_ctx->parent, parent_completion,
-                              parent_extents, NULL, &m_read_data, 0);
+                              std::move(parent_extents), nullptr, &m_read_data,
+                              0);
 }
 
 /** write **/
@@ -520,7 +523,8 @@ void AbstractAioObjectWrite::send_copyup()
   if (it == m_ictx->copyup_list.end()) {
     CopyupRequest *new_req = new CopyupRequest(m_ictx, m_oid,
                                                m_object_no,
-                                               m_parent_extents);
+                                               std::move(m_parent_extents));
+    m_parent_extents.clear();
 
     // make sure to wait on this CopyupRequest
     new_req->append_request(this);

diff --git a/src/librbd/AioObjectRequest.h b/src/librbd/AioObjectRequest.h
index 30345533ad0cea04a5ad30536d46fd4876df3609..2c7a14866cf154691407da367792b7e14668c8c3 100644 (file)
--- a/src/librbd/AioObjectRequest.h
+++ b/src/librbd/AioObjectRequest.h
@@ -168,7 +168,7 @@ private:
 
   void send_copyup();
 
-  void read_from_parent(const Extents& image_extents);
+  void read_from_parent(Extents&& image_extents);
 };
 
 class AbstractAioObjectWrite : public AioObjectRequest<> {

diff --git a/src/librbd/CopyupRequest.cc b/src/librbd/CopyupRequest.cc
index 56524d3c86fc62f6ad9d69d661f0580af89eae97..cbb04cb9606ea4ebfa90a2d6c6024d54c6464c7f 100644 (file)
--- a/src/librbd/CopyupRequest.cc
+++ b/src/librbd/CopyupRequest.cc
@@ -78,8 +78,7 @@ private:
 
 
 CopyupRequest::CopyupRequest(ImageCtx *ictx, const std::string &oid,
-                             uint64_t objectno,
-                      vector<pair<uint64_t,uint64_t> >& image_extents)
+                             uint64_t objectno, Extents &&image_extents)
   : m_ictx(ictx), m_oid(oid), m_object_no(objectno),
     m_image_extents(image_extents), m_state(STATE_READ_FROM_PARENT)
 {
@@ -188,8 +187,8 @@ void CopyupRequest::send()
                          << ", oid " << m_oid
                          << ", extents " << m_image_extents
                          << dendl;
-  AioImageRequest<>::aio_read(m_ictx->parent, comp, m_image_extents, NULL,
-                              &m_copyup_data, 0);
+  AioImageRequest<>::aio_read(m_ictx->parent, comp, std::move(m_image_extents),
+                              nullptr, &m_copyup_data, 0);
 }
 
 void CopyupRequest::complete(int r)
@@ -205,7 +204,6 @@ bool CopyupRequest::should_complete(int r)
   CephContext *cct = m_ictx->cct;
   ldout(cct, 20) << __func__ << " " << this
                  << ": oid " << m_oid
-                 << ", extents " << m_image_extents
                  << ", r " << r << dendl;
 
   uint64_t pending_copyups;

diff --git a/src/librbd/CopyupRequest.h b/src/librbd/CopyupRequest.h
index 62787681112b9f811980f2396a6eb549d4ac9678..0ae6013f69166cb662a2cecfe003a4464867bd1c 100644 (file)
--- a/src/librbd/CopyupRequest.h
+++ b/src/librbd/CopyupRequest.h
@@ -16,8 +16,10 @@ struct ImageCtx;
 
 class CopyupRequest {
 public:
+  typedef std::vector<std::pair<uint64_t, uint64_t> > Extents;
+
   CopyupRequest(ImageCtx *ictx, const std::string &oid, uint64_t objectno,
-                vector<pair<uint64_t,uint64_t> >& image_extents);
+                Extents &&image_extents);
   ~CopyupRequest();
 
   void append_request(AioObjectRequest<ImageCtx> *req);
@@ -63,7 +65,7 @@ private:
   ImageCtx *m_ictx;
   std::string m_oid;
   uint64_t m_object_no;
-  vector<pair<uint64_t,uint64_t> > m_image_extents;
+  Extents m_image_extents;
   State m_state;
   ceph::bufferlist m_copyup_data;
   vector<AioObjectRequest<ImageCtx> *> m_pending_requests;

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 8176339a2febf87d957ad317d81ed738145b9f58..d76e2a1eff6a6ecee5cd2ecdf8585e04a9182429 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -2465,7 +2465,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force,
       Context *ctx = new C_CopyRead(&throttle, dest, offset, bl);
       AioCompletion *comp = AioCompletion::create_and_start(ctx, src,
                                                             AIO_TYPE_READ);
-      AioImageRequest<>::aio_read(src, comp, offset, len, NULL, bl,
+      AioImageRequest<>::aio_read(src, comp, {{offset, len}}, nullptr, bl,
                                   fadvise_flags);
       prog_ctx.update_progress(offset, src_size);
     }
@@ -2691,7 +2691,7 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force,
       C_SaferCond ctx;
       AioCompletion *c = AioCompletion::create_and_start(&ctx, ictx,
                                                          AIO_TYPE_READ);
-      AioImageRequest<>::aio_read(ictx, c, off, read_len, NULL, &bl, 0);
+      AioImageRequest<>::aio_read(ictx, c, {{off, read_len}}, nullptr, &bl, 0);
 
       int ret = ctx.wait();
       if (ret < 0) {