From ec07a889aa1d6552181f88189a9f7017a49ad8ab Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 2 Mar 2023 15:11:05 -0500 Subject: [PATCH] rgw/aio: update Aio callers in rados and d3n Signed-off-by: Casey Bodley (cherry picked from commit d4eec9d1fbaa7031105dbeb86b60a6d2cd3de70c) Conflicts: src/rgw/rgw_d3n_cacherequest.h some d3n stuff backported earlier --- src/rgw/driver/rados/rgw_d3n_datacache.h | 10 ++++++---- src/rgw/driver/rados/rgw_putobj_processor.cc | 8 +++++--- src/rgw/driver/rados/rgw_rados.cc | 17 +++++++++++++---- src/rgw/rgw_d3n_cacherequest.h | 5 ++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/rgw/driver/rados/rgw_d3n_datacache.h b/src/rgw/driver/rados/rgw_d3n_datacache.h index feaa3f2b7c7db..e490627c05827 100644 --- a/src/rgw/driver/rados/rgw_d3n_datacache.h +++ b/src/rgw/driver/rados/rgw_d3n_datacache.h @@ -207,7 +207,8 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const const uint64_t cost = len; const uint64_t id = obj_ofs; // use logical object offset for sorting replies - auto completed = d->aio->get(obj, rgw::Aio::librados_op(std::move(op), d->yield), cost, id); + auto& ref = obj.get_ref(); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } else { ldpp_dout(dpp, 20) << "D3nDataCache::" << __func__ << "(): oid=" << read_obj.oid << ", is_head_obj=" << is_head_obj << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << ", len=" << len << dendl; @@ -225,13 +226,14 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const lsubdout(g_ceph_context, rgw, 0) << "D3nDataCache: Error: failed to open rados context for " << read_obj << ", r=" << r << dendl; return r; } + auto& ref = obj.get_ref(); const bool is_compressed = (astate->attrset.find(RGW_ATTR_COMPRESSION) != astate->attrset.end()); const bool is_encrypted = (astate->attrset.find(RGW_ATTR_CRYPT_MODE) != astate->attrset.end()); if (read_ofs != 0 || astate->size != astate->accounted_size || is_compressed || is_encrypted) { d->d3n_bypass_cache_write = true; lsubdout(g_ceph_context, rgw, 5) << "D3nDataCache: " << __func__ << "(): Note - bypassing datacache: oid=" << read_obj.oid << ", read_ofs!=0 = " << read_ofs << ", size=" << astate->size << " != accounted_size=" << astate->accounted_size << ", is_compressed=" << is_compressed << ", is_encrypted=" << is_encrypted << dendl; - auto completed = d->aio->get(obj, rgw::Aio::librados_op(std::move(op), d->yield), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); r = d->flush(std::move(completed)); return r; } @@ -239,7 +241,7 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const if (d->rgwrados->d3n_data_cache->get(oid, len)) { // Read From Cache ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): READ FROM CACHE: oid=" << read_obj.oid << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << ", len=" << len << dendl; - auto completed = d->aio->get(obj, rgw::Aio::d3n_cache_op(dpp, d->yield, read_ofs, len, d->rgwrados->d3n_data_cache->cache_location), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::d3n_cache_op(dpp, d->yield, read_ofs, len, d->rgwrados->d3n_data_cache->cache_location), cost, id); r = d->flush(std::move(completed)); if (r < 0) { lsubdout(g_ceph_context, rgw, 0) << "D3nDataCache: " << __func__ << "(): Error: failed to drain/flush, r= " << r << dendl; @@ -248,7 +250,7 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const } else { // Write To Cache ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): WRITE TO CACHE: oid=" << read_obj.oid << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << " len=" << len << dendl; - auto completed = d->aio->get(obj, rgw::Aio::librados_op(std::move(op), d->yield), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } } diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc index dc3b5c9ee6535..b73537de0723e 100644 --- a/src/rgw/driver/rados/rgw_putobj_processor.cc +++ b/src/rgw/driver/rados/rgw_putobj_processor.cc @@ -105,7 +105,7 @@ static int process_completed(const AioResultList& completed, RawObjSet *written) std::optional error; for (auto& r : completed) { if (r.result >= 0) { - written->insert(r.obj.get_ref().obj); + written->insert(r.obj); } else if (!error) { // record first error code error = r.result; } @@ -150,7 +150,8 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset) op.write(offset, data); } constexpr uint64_t id = 0; // unused - auto c = aio->get(stripe_obj, Aio::librados_op(std::move(op), y), cost, id); + auto& ref = stripe_obj.get_ref(); + auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); return process_completed(c, &written); } @@ -164,7 +165,8 @@ int RadosWriter::write_exclusive(const bufferlist& data) op.write_full(data); constexpr uint64_t id = 0; // unused - auto c = aio->get(stripe_obj, Aio::librados_op(std::move(op), y), cost, id); + auto& ref = stripe_obj.get_ref(); + auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); auto d = aio->drain(); c.splice(c.end(), d); return process_completed(c, &written); diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 566b7d6c2f9f0..9a4361ffe609a 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -4561,7 +4561,8 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx, static constexpr uint64_t cost = 1; // 1 throttle unit per request static constexpr uint64_t id = 0; // ids unused - rgw::AioResultList completed = aio->get(obj, rgw::Aio::librados_op(std::move(op), y), cost, id); + auto& ref = obj.get_ref(); + rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); ret = rgw::check_for_errors(completed); all_results.splice(all_results.end(), completed); if (ret < 0) { @@ -4628,12 +4629,19 @@ done_ret: if (r.result < 0) { continue; // skip errors } + auto obj = svc.rados->obj(r.obj); + ret2 = obj.open(dpp); + if (ret2 < 0) { + continue; + } + auto& ref = obj.get_ref(); + ObjectWriteOperation op; cls_refcount_put(op, ref_tag, true); static constexpr uint64_t cost = 1; // 1 throttle unit per request static constexpr uint64_t id = 0; // ids unused - rgw::AioResultList completed = aio->get(r.obj, rgw::Aio::librados_op(std::move(op), y), cost, id); + rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); ret2 = rgw::check_for_errors(completed); if (ret2 < 0) { ldpp_dout(dpp, 0) << "ERROR: cleanup after error failed to drop reference on obj=" << r.obj << dendl; @@ -6758,7 +6766,7 @@ int get_obj_data::flush(rgw::AioResultList&& results) { if (rgwrados->get_use_datacache()) { const std::lock_guard l(d3n_get_data.d3n_lock); - auto oid = completed.front().obj.get_ref().obj.oid; + auto oid = completed.front().obj.oid; if (bl.length() <= g_conf()->rgw_get_obj_max_req_size && !d3n_bypass_cache_write) { lsubdout(g_ceph_context, rgw_datacache, 10) << "D3nDataCache: " << __func__ << "(): bl.length <= rgw_get_obj_max_req_size (default 4MB) - write to datacache, bl.length=" << bl.length() << dendl; rgwrados->d3n_data_cache->put(bl, bl.length(), oid); @@ -6826,7 +6834,8 @@ int RGWRados::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const uint64_t cost = len; const uint64_t id = obj_ofs; // use logical object offset for sorting replies - auto completed = d->aio->get(obj, rgw::Aio::librados_op(std::move(op), d->yield), cost, id); + auto& ref = obj.get_ref(); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } diff --git a/src/rgw/rgw_d3n_cacherequest.h b/src/rgw/rgw_d3n_cacherequest.h index edc70247fe1be..c5f17810f02d2 100644 --- a/src/rgw/rgw_d3n_cacherequest.h +++ b/src/rgw/rgw_d3n_cacherequest.h @@ -137,9 +137,8 @@ struct D3nL1CacheRequest { async_completion init(yield); auto ex = get_associated_executor(init.completion_handler); - auto& ref = r.obj.get_ref(); - ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): oid=" << ref.obj.oid << dendl; - async_read(dpp, context, cache_location+"/"+url_encode(ref.obj.oid, true), read_ofs, read_len, bind_executor(ex, d3n_libaio_handler{aio, r})); + ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): oid=" << r.obj.oid << dendl; + async_read(dpp, context, cache_location+"/"+url_encode(r.obj.oid, true), read_ofs, read_len, bind_executor(ex, d3n_libaio_handler{aio, r})); } }; -- 2.39.5