From: Samuel Just Date: Sat, 7 Dec 2013 21:09:52 +0000 (-0800) Subject: ReplicatedPG: make COPY_GET async read friendly X-Git-Tag: v0.78~286^2~16 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b3c2ef2d8f8f2e08350412430e2e1417a9b19111;p=ceph.git ReplicatedPG: make COPY_GET async read friendly Signed-off-by: Samuel Just --- diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index d0c9338a98e7..87bbe7dce538 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -4241,14 +4241,14 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops) case CEPH_OSD_OP_COPY_GET_CLASSIC: ++ctx->num_read; - result = fill_in_copy_get(bp, osd_op, ctx->obc, true); + result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, true); if (result == -EINVAL) goto fail; break; case CEPH_OSD_OP_COPY_GET: ++ctx->num_read; - result = fill_in_copy_get(bp, osd_op, ctx->obc, false); + result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, false); if (result == -EINVAL) goto fail; break; @@ -5049,8 +5049,27 @@ struct C_Copyfrom : public Context { } }; -int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op, - ObjectContextRef& obc, bool classic) +struct C_CopyFrom_AsyncReadCb : public Context { + OSDOp *osd_op; + object_copy_data_t reply_obj; + bool classic; + C_CopyFrom_AsyncReadCb(OSDOp *osd_op, bool classic) : + osd_op(osd_op), classic(classic) {} + void finish(int r) { + if (classic) { + reply_obj.encode_classic(osd_op->outdata); + } else { + ::encode(reply_obj, osd_op->outdata); + } + } +}; + +int ReplicatedPG::fill_in_copy_get( + OpContext *ctx, + bufferlist::iterator& bp, + OSDOp& osd_op, + ObjectContextRef &obc, + bool classic) { object_info_t& oi = obc->obs.oi; hobject_t& soid = oi.soid; @@ -5066,7 +5085,13 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op, return result; } - object_copy_data_t reply_obj; + bool async_read_started = false; + object_copy_data_t _reply_obj; + C_CopyFrom_AsyncReadCb *cb = NULL; + if (pool.info.ec_pool()) { + cb = new C_CopyFrom_AsyncReadCb(&osd_op, classic); + } + object_copy_data_t &reply_obj = cb ? cb->reply_obj : _reply_obj; // size, mtime reply_obj.size = oi.size; reply_obj.mtime = oi.mtime; @@ -5094,10 +5119,19 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op, bufferlist& bl = reply_obj.data; if (left > 0 && !cursor.data_complete) { if (cursor.data_offset < oi.size) { - result = pgbackend->objects_read_sync( - oi.soid, cursor.data_offset, left, &bl); - if (result < 0) - return result; + if (cb) { + async_read_started = true; + ctx->pending_async_reads.push_back( + make_pair( + make_pair(cursor.data_offset, left), + make_pair(&bl, cb))); + result = MIN(oi.size - cursor.data_offset, (uint64_t)left); + } else { + result = pgbackend->objects_read_sync( + oi.soid, cursor.data_offset, left, &bl); + if (result < 0) + return result; + } assert(result <= left); left -= result; cursor.data_offset += result; @@ -5111,26 +5145,30 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op, // omap std::map& out_omap = reply_obj.omap; - if (left > 0 && !cursor.omap_complete) { - assert(cursor.data_complete); - if (cursor.omap_offset.empty()) { - osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header); - } - ObjectMap::ObjectMapIterator iter = - osd->store->get_omap_iterator(coll, oi.soid); - assert(iter); - iter->upper_bound(cursor.omap_offset); - if (iter->valid()) { - for (; left > 0 && iter->valid(); iter->next()) { - out_omap.insert(make_pair(iter->key(), iter->value())); - left -= iter->key().length() + 4 + iter->value().length() + 4; + if (pool.info.ec_pool()) { + cursor.omap_complete = true; + } else { + if (left > 0 && !cursor.omap_complete) { + assert(cursor.data_complete); + if (cursor.omap_offset.empty()) { + osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header); + } + ObjectMap::ObjectMapIterator iter = + osd->store->get_omap_iterator(coll, oi.soid); + assert(iter); + iter->upper_bound(cursor.omap_offset); + if (iter->valid()) { + for (; left > 0 && iter->valid(); iter->next()) { + out_omap.insert(make_pair(iter->key(), iter->value())); + left -= iter->key().length() + 4 + iter->value().length() + 4; + } + } + if (iter->valid()) { + cursor.omap_offset = iter->key(); + } else { + cursor.omap_complete = true; + dout(20) << " got omap" << dendl; } - } - if (iter->valid()) { - cursor.omap_offset = iter->key(); - } else { - cursor.omap_complete = true; - dout(20) << " got omap" << dendl; } } @@ -5141,14 +5179,15 @@ int ReplicatedPG::fill_in_copy_get(bufferlist::iterator& bp, OSDOp& osd_op, << " " << out_omap.size() << " keys" << dendl; reply_obj.cursor = cursor; - if (classic) { - if (reply_obj.omap_header.length() > 0) { - derr << oi.soid << " omap header being dropped by classic copy-get api" - << dendl; + if (!async_read_started) { + if (classic) { + reply_obj.encode_classic(osd_op.outdata); + } else { + ::encode(reply_obj, osd_op.outdata); } - reply_obj.encode_classic(osd_op.outdata); - } else { - ::encode(reply_obj, osd_op.outdata); + } + if (cb && !async_read_started) { + delete cb; } result = 0; return result; diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 30dc09ffc80c..2057f7d9d79e 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -1003,8 +1003,13 @@ protected: // -- copyfrom -- map copy_ops; - int fill_in_copy_get(bufferlist::iterator& bp, OSDOp& op, - ObjectContextRef& obc, bool classic); + int fill_in_copy_get( + OpContext *ctx, + bufferlist::iterator& bp, + OSDOp& op, + ObjectContextRef& obc, + bool classic); + /** * To copy an object, call start_copy. *