rgw/aio: librados_op() requires librados::IoCtx

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index afab8b75b0bd52caf013dc589152ef54349dd495..0bbc1f7bfa8bde2b43eb82975b883016e801e929 100644 (file)
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -28,42 +28,49 @@ void cb(librados::completion_t, void* arg);
 
 struct state {
   Aio* aio;
+  librados::IoCtx ctx;
   librados::AioCompletion* c;
 
-  state(Aio* aio, AioResult& r)
-    : aio(aio),
+  state(Aio* aio, librados::IoCtx ctx, AioResult& r)
+    : aio(aio), ctx(std::move(ctx)),
       c(librados::Rados::aio_create_completion(&r, &cb)) {}
 };
 
 void cb(librados::completion_t, void* arg) {
   static_assert(sizeof(AioResult::user_data) >= sizeof(state));
-  static_assert(std::is_trivially_destructible_v<state>);
   auto& r = *(static_cast<AioResult*>(arg));
   auto s = reinterpret_cast<state*>(&r.user_data);
   r.result = s->c->get_return_value();
   s->c->release();
-  s->aio->put(r);
+  Aio* aio = s->aio;
+  // manually destroy the state that was constructed with placement new
+  s->~state();
+  aio->put(r);
 }
 
 template <typename Op>
-Aio::OpFunc aio_abstract(Op&& op) {
-  return [op = std::move(op)] (Aio* aio, AioResult& r) mutable {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op) {
+  return [ctx = std::move(ctx), op = std::move(op)] (Aio* aio, AioResult& r) mutable {
       constexpr bool read = std::is_same_v<std::decay_t<Op>, librados::ObjectReadOperation>;
-      auto s = new (&r.user_data) state(aio, r);
+      // use placement new to construct the rados state inside of user_data
+      auto s = new (&r.user_data) state(aio, ctx, r);
       if constexpr (read) {
-        r.result = r.obj.aio_operate(s->c, &op, &r.data);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data);
       } else {
-        r.result = r.obj.aio_operate(s->c, &op);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op);
       }
       if (r.result < 0) {
+        // cb() won't be called, so release everything here
         s->c->release();
         aio->put(r);
+        s->~state();
       }
     };
 }
 
 struct Handler {
   Aio* throttle = nullptr;
+  librados::IoCtx ctx;
   AioResult& r;
   // write callback
   void operator()(boost::system::error_code ec) const {
@@ -79,18 +86,18 @@ struct Handler {
 };
 
 template <typename Op>
-Aio::OpFunc aio_abstract(Op&& op, boost::asio::io_context& context,
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
+                         boost::asio::io_context& context,
                          yield_context yield) {
-  return [op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
+  return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
       // arrange for the completion Handler to run on the yield_context's strand
       // executor so it can safely call back into Aio without locking
       using namespace boost::asio;
       async_completion<yield_context, void()> init(yield);
       auto ex = get_associated_executor(init.completion_handler);
 
-      auto& ref = r.obj.get_ref();
-      librados::async_operate(context, ref.pool.ioctx(), ref.obj.oid, &op, 0,
-                              bind_executor(ex, Handler{aio, r}));
+      librados::async_operate(context, ctx, r.obj.oid, &op, 0,
+                              bind_executor(ex, Handler{aio, ctx, r}));
     };
 }
 
@@ -99,35 +106,36 @@ Aio::OpFunc d3n_cache_aio_abstract(const DoutPrefixProvider *dpp, optional_yield
   return [dpp, y, read_ofs, read_len, location] (Aio* aio, AioResult& r) mutable {
     // d3n data cache requires yield context (rgw_beast_enable_async=true)
     ceph_assert(y);
-    auto& ref = r.obj.get_ref();
     auto c = std::make_unique<D3nL1CacheRequest>();
-    lsubdout(g_ceph_context, rgw_datacache, 20) << "D3nDataCache: d3n_cache_aio_abstract(): libaio Read From Cache, oid=" << ref.obj.oid << dendl;
+    lsubdout(g_ceph_context, rgw_datacache, 20) << "D3nDataCache: d3n_cache_aio_abstract(): libaio Read From Cache, oid=" << r.obj.oid << dendl;
     c->file_aio_read_abstract(dpp, y.get_io_context(), y.get_yield_context(), location, read_ofs, read_len, aio, r);
   };
 }
 
 
 template <typename Op>
-Aio::OpFunc aio_abstract(Op&& op, optional_yield y) {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y) {
   static_assert(std::is_base_of_v<librados::ObjectOperation, std::decay_t<Op>>);
   static_assert(!std::is_lvalue_reference_v<Op>);
   static_assert(!std::is_const_v<Op>);
   if (y) {
-    return aio_abstract(std::forward<Op>(op), y.get_io_context(),
-                        y.get_yield_context());
+    return aio_abstract(std::move(ctx), std::forward<Op>(op),
+                        y.get_io_context(), y.get_yield_context());
   }
-  return aio_abstract(std::forward<Op>(op));
+  return aio_abstract(std::move(ctx), std::forward<Op>(op));
 }
 
 } // anonymous namespace
 
-Aio::OpFunc Aio::librados_op(librados::ObjectReadOperation&& op,
+Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
+                             librados::ObjectReadOperation&& op,
                              optional_yield y) {
-  return aio_abstract(std::move(op), y);
+  return aio_abstract(std::move(ctx), std::move(op), y);
 }
-Aio::OpFunc Aio::librados_op(librados::ObjectWriteOperation&& op,
+Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
+                             librados::ObjectWriteOperation&& op,
                              optional_yield y) {
-  return aio_abstract(std::move(op), y);
+  return aio_abstract(std::move(ctx), std::move(op), y);
 }
 
 Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,

diff --git a/src/rgw/rgw_aio.h b/src/rgw/rgw_aio.h
index a2c539c17ef00024f9135e5859d9cd132d7b9f6a..6a49a5370eb5dd45bfc66ae173dcdbc81eba0f17 100644 (file)
--- a/src/rgw/rgw_aio.h
+++ b/src/rgw/rgw_aio.h
@@ -93,9 +93,11 @@ class Aio {
   // wait for all outstanding completions and return their results
   virtual AioResultList drain() = 0;
 
-  static OpFunc librados_op(librados::ObjectReadOperation&& op,
+  static OpFunc librados_op(librados::IoCtx ctx,
+                            librados::ObjectReadOperation&& op,
                             optional_yield y);
-  static OpFunc librados_op(librados::ObjectWriteOperation&& op,
+  static OpFunc librados_op(librados::IoCtx ctx,
+                            librados::ObjectWriteOperation&& op,
                             optional_yield y);
   static OpFunc d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
                              off_t read_ofs, off_t read_len, std::string& location);