ReplicatedPG: make COPY_GET async read friendly

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index d0c9338a98e7c6310ecbecc7babdd668f7d8f075..87bbe7dce53866c8531378737526e4288ad0fbc9 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -4241,14 +4241,14 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
 
     case CEPH_OSD_OP_COPY_GET_CLASSIC:
       ++ctx->num_read;
-      result = fill_in_copy_get(bp, osd_op, ctx->obc, true);
+      result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, true);
       if (result == -EINVAL)
        goto fail;
       break;
 
     case CEPH_OSD_OP_COPY_GET:
       ++ctx->num_read;
-      result = fill_in_copy_get(bp, osd_op, ctx->obc, false);
+      result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, false);
       if (result == -EINVAL)
        goto fail;
       break;
@@ -5049,8 +5049,27 @@ struct C_Copyfrom : public Context {
   }
 };
 
-int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
-                                   ObjectContextRef& obc, bool classic)
+struct C_CopyFrom_AsyncReadCb : public Context {
+  OSDOp *osd_op;
+  object_copy_data_t reply_obj;
+  bool classic;
+  C_CopyFrom_AsyncReadCb(OSDOp *osd_op, bool classic) :
+    osd_op(osd_op), classic(classic) {}
+  void finish(int r) {
+    if (classic) {
+      reply_obj.encode_classic(osd_op->outdata);
+    } else {
+      ::encode(reply_obj, osd_op->outdata);
+    }
+  }
+};
+
+int ReplicatedPG::fill_in_copy_get(
+  OpContext *ctx,
+  bufferlist::iterator& bp,
+  OSDOp& osd_op,
+  ObjectContextRef &obc,
+  bool classic)
 {
   object_info_t& oi = obc->obs.oi;
   hobject_t& soid = oi.soid;
@@ -5066,7 +5085,13 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
     return result;
   }
 
-  object_copy_data_t reply_obj;
+  bool async_read_started = false;
+  object_copy_data_t _reply_obj;
+  C_CopyFrom_AsyncReadCb *cb = NULL;
+  if (pool.info.ec_pool()) {
+    cb = new C_CopyFrom_AsyncReadCb(&osd_op, classic);
+  }
+  object_copy_data_t &reply_obj = cb ? cb->reply_obj : _reply_obj;
   // size, mtime
   reply_obj.size = oi.size;
   reply_obj.mtime = oi.mtime;
@@ -5094,10 +5119,19 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
   bufferlist& bl = reply_obj.data;
   if (left > 0 && !cursor.data_complete) {
     if (cursor.data_offset < oi.size) {
-      result = pgbackend->objects_read_sync(
-       oi.soid, cursor.data_offset, left, &bl);
-      if (result < 0)
-       return result;
+      if (cb) {
+       async_read_started = true;
+       ctx->pending_async_reads.push_back(
+         make_pair(
+           make_pair(cursor.data_offset, left),
+           make_pair(&bl, cb)));
+       result = MIN(oi.size - cursor.data_offset, (uint64_t)left);
+      } else {
+       result = pgbackend->objects_read_sync(
+         oi.soid, cursor.data_offset, left, &bl);
+       if (result < 0)
+         return result;
+      }
       assert(result <= left);
       left -= result;
       cursor.data_offset += result;
@@ -5111,26 +5145,30 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
 
   // omap
   std::map<std::string,bufferlist>& out_omap = reply_obj.omap;
-  if (left > 0 && !cursor.omap_complete) {
-    assert(cursor.data_complete);
-    if (cursor.omap_offset.empty()) {
-      osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header);
-    }
-    ObjectMap::ObjectMapIterator iter =
-      osd->store->get_omap_iterator(coll, oi.soid);
-    assert(iter);
-    iter->upper_bound(cursor.omap_offset);
-    if (iter->valid()) {
-      for (; left > 0 && iter->valid(); iter->next()) {
-       out_omap.insert(make_pair(iter->key(), iter->value()));
-       left -= iter->key().length() + 4 + iter->value().length() + 4;
+  if (pool.info.ec_pool()) {
+    cursor.omap_complete = true;
+  } else {
+    if (left > 0 && !cursor.omap_complete) {
+      assert(cursor.data_complete);
+      if (cursor.omap_offset.empty()) {
+       osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header);
+      }
+      ObjectMap::ObjectMapIterator iter =
+       osd->store->get_omap_iterator(coll, oi.soid);
+      assert(iter);
+      iter->upper_bound(cursor.omap_offset);
+      if (iter->valid()) {
+       for (; left > 0 && iter->valid(); iter->next()) {
+         out_omap.insert(make_pair(iter->key(), iter->value()));
+         left -= iter->key().length() + 4 + iter->value().length() + 4;
+       }
+      }
+      if (iter->valid()) {
+       cursor.omap_offset = iter->key();
+      } else {
+       cursor.omap_complete = true;
+       dout(20) << " got omap" << dendl;
       }
-    }
-    if (iter->valid()) {
-      cursor.omap_offset = iter->key();
-    } else {
-      cursor.omap_complete = true;
-      dout(20) << " got omap" << dendl;
     }
   }
 
@@ -5141,14 +5179,15 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op,
           << " " << out_omap.size() << " keys"
           << dendl;
   reply_obj.cursor = cursor;
-  if (classic) {
-    if (reply_obj.omap_header.length() > 0) {
-      derr << oi.soid << " omap header being dropped by classic copy-get api"
-          << dendl;
+  if (!async_read_started) {
+    if (classic) {
+      reply_obj.encode_classic(osd_op.outdata);
+    } else {
+      ::encode(reply_obj, osd_op.outdata);
     }
-    reply_obj.encode_classic(osd_op.outdata);
-  } else {
-    ::encode(reply_obj, osd_op.outdata);
+  }
+  if (cb && !async_read_started) {
+    delete cb;
   }
   result = 0;
   return result;

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 30dc09ffc80c0aa18150e2263473231086f6df2d..2057f7d9d79e37128a3cad34da4d0eb375c89f03 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -1003,8 +1003,13 @@ protected:
   // -- copyfrom --
   map<hobject_t, CopyOpRef> copy_ops;
 
-  int fill_in_copy_get(bufferlist::iterator& bp, OSDOp& op,
-                       ObjectContextRef& obc, bool classic);
+  int fill_in_copy_get(
+    OpContext *ctx,
+    bufferlist::iterator& bp,
+    OSDOp& op,
+    ObjectContextRef& obc,
+    bool classic);
+
   /**
    * To copy an object, call start_copy.
    *