From: Casey Bodley Date: Thu, 2 Mar 2023 15:06:06 +0000 (-0500) Subject: rgw/aio: librados_op() requires librados::IoCtx X-Git-Tag: testing/wip-mchangir-testing-20241122.051535-debug~7^2~7 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=8d9f03a61e6f7e8e9f02bb5e609be9bd197b237a;p=ceph-ci.git rgw/aio: librados_op() requires librados::IoCtx Signed-off-by: Casey Bodley (cherry picked from commit 1bc8d66495cfa46b4624a0524b3443e56ebc96d0) Conflicts: src/rgw/rgw_aio.cc some d3n stuff backported earlier --- diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc index 4fba513b836..02e3411858a 100644 --- a/src/rgw/rgw_aio.cc +++ b/src/rgw/rgw_aio.cc @@ -28,42 +28,49 @@ void cb(librados::completion_t, void* arg); struct state { Aio* aio; + librados::IoCtx ctx; librados::AioCompletion* c; - state(Aio* aio, AioResult& r) - : aio(aio), + state(Aio* aio, librados::IoCtx ctx, AioResult& r) + : aio(aio), ctx(std::move(ctx)), c(librados::Rados::aio_create_completion(&r, &cb)) {} }; void cb(librados::completion_t, void* arg) { static_assert(sizeof(AioResult::user_data) >= sizeof(state)); - static_assert(std::is_trivially_destructible_v); auto& r = *(static_cast(arg)); auto s = reinterpret_cast(&r.user_data); r.result = s->c->get_return_value(); s->c->release(); - s->aio->put(r); + Aio* aio = s->aio; + // manually destroy the state that was constructed with placement new + s->~state(); + aio->put(r); } template -Aio::OpFunc aio_abstract(Op&& op) { - return [op = std::move(op)] (Aio* aio, AioResult& r) mutable { +Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op) { + return [ctx = std::move(ctx), op = std::move(op)] (Aio* aio, AioResult& r) mutable { constexpr bool read = std::is_same_v, librados::ObjectReadOperation>; - auto s = new (&r.user_data) state(aio, r); + // use placement new to construct the rados state inside of user_data + auto s = new (&r.user_data) state(aio, ctx, r); if constexpr (read) { - r.result = r.obj.aio_operate(s->c, &op, &r.data); + r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data); } else { - r.result = r.obj.aio_operate(s->c, &op); + r.result = ctx.aio_operate(r.obj.oid, s->c, &op); } if (r.result < 0) { + // cb() won't be called, so release everything here s->c->release(); aio->put(r); + s->~state(); } }; } struct Handler { Aio* throttle = nullptr; + librados::IoCtx ctx; AioResult& r; // write callback void operator()(boost::system::error_code ec) const { @@ -79,18 +86,18 @@ struct Handler { }; template -Aio::OpFunc aio_abstract(Op&& op, boost::asio::io_context& context, +Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, + boost::asio::io_context& context, yield_context yield) { - return [op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable { + return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable { // arrange for the completion Handler to run on the yield_context's strand // executor so it can safely call back into Aio without locking using namespace boost::asio; async_completion init(yield); auto ex = get_associated_executor(init.completion_handler); - auto& ref = r.obj.get_ref(); - librados::async_operate(context, ref.pool.ioctx(), ref.obj.oid, &op, 0, - bind_executor(ex, Handler{aio, r})); + librados::async_operate(context, ctx, r.obj.oid, &op, 0, + bind_executor(ex, Handler{aio, ctx, r})); }; } @@ -99,35 +106,36 @@ Aio::OpFunc d3n_cache_aio_abstract(const DoutPrefixProvider *dpp, optional_yield return [dpp, y, read_ofs, read_len, cache_location] (Aio* aio, AioResult& r) mutable { // d3n data cache requires yield context (rgw_beast_enable_async=true) ceph_assert(y); - auto& ref = r.obj.get_ref(); auto c = std::make_unique(); - lsubdout(g_ceph_context, rgw_datacache, 20) << "D3nDataCache: d3n_cache_aio_abstract(): libaio Read From Cache, oid=" << ref.obj.oid << dendl; + lsubdout(g_ceph_context, rgw_datacache, 20) << "D3nDataCache: d3n_cache_aio_abstract(): libaio Read From Cache, oid=" << r.obj.oid << dendl; c->file_aio_read_abstract(dpp, y.get_io_context(), y.get_yield_context(), cache_location, read_ofs, read_len, aio, r); }; } template -Aio::OpFunc aio_abstract(Op&& op, optional_yield y) { +Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y) { static_assert(std::is_base_of_v>); static_assert(!std::is_lvalue_reference_v); static_assert(!std::is_const_v); if (y) { - return aio_abstract(std::forward(op), y.get_io_context(), - y.get_yield_context()); + return aio_abstract(std::move(ctx), std::forward(op), + y.get_io_context(), y.get_yield_context()); } - return aio_abstract(std::forward(op)); + return aio_abstract(std::move(ctx), std::forward(op)); } } // anonymous namespace -Aio::OpFunc Aio::librados_op(librados::ObjectReadOperation&& op, +Aio::OpFunc Aio::librados_op(librados::IoCtx ctx, + librados::ObjectReadOperation&& op, optional_yield y) { - return aio_abstract(std::move(op), y); + return aio_abstract(std::move(ctx), std::move(op), y); } -Aio::OpFunc Aio::librados_op(librados::ObjectWriteOperation&& op, +Aio::OpFunc Aio::librados_op(librados::IoCtx ctx, + librados::ObjectWriteOperation&& op, optional_yield y) { - return aio_abstract(std::move(op), y); + return aio_abstract(std::move(ctx), std::move(op), y); } Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y, diff --git a/src/rgw/rgw_aio.h b/src/rgw/rgw_aio.h index a2c539c17ef..6a49a5370eb 100644 --- a/src/rgw/rgw_aio.h +++ b/src/rgw/rgw_aio.h @@ -93,9 +93,11 @@ class Aio { // wait for all outstanding completions and return their results virtual AioResultList drain() = 0; - static OpFunc librados_op(librados::ObjectReadOperation&& op, + static OpFunc librados_op(librados::IoCtx ctx, + librados::ObjectReadOperation&& op, optional_yield y); - static OpFunc librados_op(librados::ObjectWriteOperation&& op, + static OpFunc librados_op(librados::IoCtx ctx, + librados::ObjectWriteOperation&& op, optional_yield y); static OpFunc d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y, off_t read_ofs, off_t read_len, std::string& location);