From 8476a02bed811eb9a1ca751eaed014ca312eef5e Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 5 Jul 2024 18:36:02 -0400 Subject: [PATCH] cls/rgw: define lc ops in terms of ObjectOperation instead of IoCtx clean up the lc functions that were supposed to be hidden by CLS_CLIENT_HIDE_IOCTX this allows rgw to use them asynchonously with rgw_rados_operate() and optional_yield, and warn about blocking calls that should be async Signed-off-by: Casey Bodley --- src/cls/rgw/cls_rgw_client.cc | 102 ++++++++++++-------------- src/cls/rgw/cls_rgw_client.h | 26 +++---- src/rgw/driver/rados/rgw_sal_rados.cc | 82 +++++++++++++++++---- 3 files changed, 128 insertions(+), 82 deletions(-) diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index e65dedf14e42..57637d8c9852 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -940,13 +940,14 @@ void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const vector& op.exec(RGW_CLASS, RGW_GC_REMOVE, in); } -int cls_rgw_lc_get_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head) +void cls_rgw_lc_get_head(ObjectReadOperation& op, bufferlist& out) { - bufferlist in, out; - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_HEAD, in, out); - if (r < 0) - return r; + bufferlist in; + op.exec(RGW_CLASS, RGW_LC_GET_HEAD, in, &out, nullptr); +} +int cls_rgw_lc_get_head_decode(const bufferlist& out, cls_rgw_lc_obj_head& head) +{ cls_rgw_lc_get_head_ret ret; try { auto iter = out.cbegin(); @@ -954,32 +955,32 @@ int cls_rgw_lc_get_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& h } catch (ceph::buffer::error& err) { return -EIO; } - head = ret.head; + head = std::move(ret.head); - return r; + return 0; } -int cls_rgw_lc_put_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head) +void cls_rgw_lc_put_head(ObjectWriteOperation& op, const cls_rgw_lc_obj_head& head) { - bufferlist in, out; + bufferlist in; cls_rgw_lc_put_head_op call; call.head = head; encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_PUT_HEAD, in, out); - return r; + op.exec(RGW_CLASS, RGW_LC_PUT_HEAD, in); } -int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, const string& marker, - cls_rgw_lc_entry& entry) +void cls_rgw_lc_get_next_entry(ObjectReadOperation& op, const string& marker, + bufferlist& out) { - bufferlist in, out; + bufferlist in; cls_rgw_lc_get_next_entry_op call; call.marker = marker; encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_NEXT_ENTRY, in, out); - if (r < 0) - return r; + op.exec(RGW_CLASS, RGW_LC_GET_NEXT_ENTRY, in, &out, nullptr); +} +int cls_rgw_lc_get_next_entry_decode(const bufferlist& out, cls_rgw_lc_entry& entry) +{ cls_rgw_lc_get_next_entry_ret ret; try { auto iter = out.cbegin(); @@ -987,45 +988,42 @@ int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, const string& ma } catch (ceph::buffer::error& err) { return -EIO; } - entry = ret.entry; + entry = std::move(ret.entry); - return r; + return 0; } -int cls_rgw_lc_rm_entry(IoCtx& io_ctx, const string& oid, - const cls_rgw_lc_entry& entry) +void cls_rgw_lc_rm_entry(ObjectWriteOperation& op, + const cls_rgw_lc_entry& entry) { - bufferlist in, out; + bufferlist in; cls_rgw_lc_rm_entry_op call; call.entry = entry; encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_RM_ENTRY, in, out); - return r; + op.exec(RGW_CLASS, RGW_LC_RM_ENTRY, in); } -int cls_rgw_lc_set_entry(IoCtx& io_ctx, const string& oid, - const cls_rgw_lc_entry& entry) +void cls_rgw_lc_set_entry(ObjectWriteOperation& op, + const cls_rgw_lc_entry& entry) { bufferlist in, out; cls_rgw_lc_set_entry_op call; call.entry = entry; encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_SET_ENTRY, in, out); - return r; + op.exec(RGW_CLASS, RGW_LC_SET_ENTRY, in); } -int cls_rgw_lc_get_entry(IoCtx& io_ctx, const string& oid, - const std::string& marker, cls_rgw_lc_entry& entry) +void cls_rgw_lc_get_entry(ObjectReadOperation& op, const std::string& marker, + bufferlist& out) { - bufferlist in, out; - cls_rgw_lc_get_entry_op call{marker};; + bufferlist in; + cls_rgw_lc_get_entry_op call{marker}; encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_ENTRY, in, out); - - if (r < 0) { - return r; - } + op.exec(RGW_CLASS, RGW_LC_GET_ENTRY, in, &out, nullptr); +} +int cls_rgw_lc_get_entry_decode(const bufferlist& out, cls_rgw_lc_entry& entry) +{ cls_rgw_lc_get_entry_ret ret; try { auto iter = out.cbegin(); @@ -1035,28 +1033,24 @@ int cls_rgw_lc_get_entry(IoCtx& io_ctx, const string& oid, } entry = std::move(ret.entry); - return r; + return 0; } -int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid, - const string& marker, - uint32_t max_entries, - vector& entries) +void cls_rgw_lc_list(ObjectReadOperation& op, const string& marker, + uint32_t max_entries, bufferlist& out) { - bufferlist in, out; - cls_rgw_lc_list_entries_op op; - - entries.clear(); - - op.marker = marker; - op.max_entries = max_entries; + bufferlist in; + cls_rgw_lc_list_entries_op call; + call.marker = marker; + call.max_entries = max_entries; - encode(op, in); + encode(call, in); - int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_LIST_ENTRIES, in, out); - if (r < 0) - return r; + op.exec(RGW_CLASS, RGW_LC_LIST_ENTRIES, in, &out, nullptr); +} +int cls_rgw_lc_list_decode(const bufferlist& out, std::vector& entries) +{ cls_rgw_lc_list_entries_ret ret; try { auto iter = out.cbegin(); @@ -1069,7 +1063,7 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid, [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b) { return a.bucket < b.bucket; }); entries = std::move(ret.entries); - return r; + return 0; } void cls_rgw_mp_upload_part_info_update(librados::ObjectWriteOperation& op, diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h index 365a51fb5d59..ab2f4c9ca851 100644 --- a/src/cls/rgw/cls_rgw_client.h +++ b/src/cls/rgw/cls_rgw_client.h @@ -609,19 +609,19 @@ int cls_rgw_gc_list(librados::IoCtx& io_ctx, std::string& oid, std::string& mark #endif /* lifecycle */ -// these overloads which call io_ctx.operate() should not be called in the rgw. -// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate() -#ifndef CLS_CLIENT_HIDE_IOCTX -int cls_rgw_lc_get_head(librados::IoCtx& io_ctx, const std::string& oid, cls_rgw_lc_obj_head& head); -int cls_rgw_lc_put_head(librados::IoCtx& io_ctx, const std::string& oid, cls_rgw_lc_obj_head& head); -int cls_rgw_lc_get_next_entry(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, cls_rgw_lc_entry& entry); -int cls_rgw_lc_rm_entry(librados::IoCtx& io_ctx, const std::string& oid, const cls_rgw_lc_entry& entry); -int cls_rgw_lc_set_entry(librados::IoCtx& io_ctx, const std::string& oid, const cls_rgw_lc_entry& entry); -int cls_rgw_lc_get_entry(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, cls_rgw_lc_entry& entry); -int cls_rgw_lc_list(librados::IoCtx& io_ctx, const std::string& oid, - const std::string& marker, uint32_t max_entries, - std::vector& entries); -#endif +void cls_rgw_lc_get_head(librados::ObjectReadOperation& op, bufferlist& bl); +int cls_rgw_lc_get_head_decode(const bufferlist& bl, cls_rgw_lc_obj_head& head); +void cls_rgw_lc_put_head(librados::ObjectWriteOperation& op, const cls_rgw_lc_obj_head& head); +void cls_rgw_lc_get_next_entry(librados::ObjectReadOperation& op, const std::string& marker, bufferlist& bl); +int cls_rgw_lc_get_next_entry_decode(const bufferlist& bl, cls_rgw_lc_entry& entry); +void cls_rgw_lc_rm_entry(librados::ObjectWriteOperation& op, const cls_rgw_lc_entry& entry); +void cls_rgw_lc_set_entry(librados::ObjectWriteOperation& op, const cls_rgw_lc_entry& entry); +void cls_rgw_lc_get_entry(librados::ObjectReadOperation& op, const std::string& marker, bufferlist& bl); +int cls_rgw_lc_get_entry_decode(const bufferlist& bl, cls_rgw_lc_entry& entry); +void cls_rgw_lc_list(librados::ObjectReadOperation& op, + const std::string& marker, uint32_t max_entries, + bufferlist& bl); +int cls_rgw_lc_list_decode(const bufferlist& bl, std::vector& entries); /* multipart */ void cls_rgw_mp_upload_part_info_update(librados::ObjectWriteOperation& op, const std::string& part_key, const RGWUploadPartInfo& info); diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 33e667ba974d..a86c164d6cbd 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -3608,10 +3608,21 @@ int RadosLifecycle::get_entry(const DoutPrefixProvider* dpp, optional_yield y, const std::string& oid, const std::string& marker, std::unique_ptr* entry) { + librados::ObjectReadOperation op; + bufferlist bl; + cls_rgw_lc_get_entry(op, marker, bl); + + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y); + if (ret < 0) { + return ret; + } + cls_rgw_lc_entry cls_entry; - int ret = cls_rgw_lc_get_entry(*store->getRados()->get_lc_pool_ctx(), oid, marker, cls_entry); - if (ret) + ret = cls_rgw_lc_get_entry_decode(bl, cls_entry); + if (ret < 0) { return ret; + } *entry = std::make_unique(cls_entry.bucket, cls_entry.start_time, cls_entry.status); return 0; @@ -3621,12 +3632,21 @@ int RadosLifecycle::get_next_entry(const DoutPrefixProvider* dpp, optional_yield const std::string& oid, const std::string& marker, std::unique_ptr* entry) { - cls_rgw_lc_entry cls_entry; - int ret = cls_rgw_lc_get_next_entry(*store->getRados()->get_lc_pool_ctx(), oid, marker, - cls_entry); + librados::ObjectReadOperation op; + bufferlist bl; + cls_rgw_lc_get_next_entry(op, marker, bl); - if (ret) + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y); + if (ret < 0) { return ret; + } + + cls_rgw_lc_entry cls_entry; + ret = cls_rgw_lc_get_next_entry_decode(bl, cls_entry); + if (ret < 0) { + return ret; + } *entry = std::make_unique(cls_entry.bucket, cls_entry.start_time, cls_entry.status); return 0; @@ -3641,7 +3661,11 @@ int RadosLifecycle::set_entry(const DoutPrefixProvider* dpp, optional_yield y, cls_entry.start_time = entry.get_start_time(); cls_entry.status = entry.get_status(); - return cls_rgw_lc_set_entry(*store->getRados()->get_lc_pool_ctx(), oid, cls_entry); + librados::ObjectWriteOperation op; + cls_rgw_lc_set_entry(op, cls_entry); + + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + return rgw_rados_operate(dpp, ioctx, oid, &op, y); } int RadosLifecycle::list_entries(const DoutPrefixProvider* dpp, optional_yield y, @@ -3650,11 +3674,21 @@ int RadosLifecycle::list_entries(const DoutPrefixProvider* dpp, optional_yield y { entries.clear(); - vector cls_entries; - int ret = cls_rgw_lc_list(*store->getRados()->get_lc_pool_ctx(), oid, marker, max_entries, cls_entries); + librados::ObjectReadOperation op; + bufferlist bl; + cls_rgw_lc_list(op, marker, max_entries, bl); - if (ret < 0) + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y); + if (ret < 0) { + return ret; + } + + vector cls_entries; + ret = cls_rgw_lc_list_decode(bl, cls_entries); + if (ret < 0) { return ret; + } for (auto& entry : cls_entries) { entries.push_back(std::make_unique(entry.bucket, oid, @@ -3668,21 +3702,35 @@ int RadosLifecycle::rm_entry(const DoutPrefixProvider* dpp, optional_yield y, const std::string& oid, LCEntry& entry) { cls_rgw_lc_entry cls_entry; - cls_entry.bucket = entry.get_bucket(); cls_entry.start_time = entry.get_start_time(); cls_entry.status = entry.get_status(); - return cls_rgw_lc_rm_entry(*store->getRados()->get_lc_pool_ctx(), oid, cls_entry); + librados::ObjectWriteOperation op; + cls_rgw_lc_rm_entry(op, cls_entry); + + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + return rgw_rados_operate(dpp, ioctx, oid, &op, y); } int RadosLifecycle::get_head(const DoutPrefixProvider* dpp, optional_yield y, const std::string& oid, std::unique_ptr* head) { + librados::ObjectReadOperation op; + bufferlist bl; + cls_rgw_lc_get_head(op, bl); + + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y); + if (ret < 0) { + return ret; + } + cls_rgw_lc_obj_head cls_head; - int ret = cls_rgw_lc_get_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head); - if (ret) + ret = cls_rgw_lc_get_head_decode(bl, cls_head); + if (ret < 0) { return ret; + } *head = std::make_unique(cls_head.start_date, cls_head.shard_rollover_date, cls_head.marker); return 0; @@ -3697,7 +3745,11 @@ int RadosLifecycle::put_head(const DoutPrefixProvider* dpp, optional_yield y, cls_head.start_date = head.get_start_date(); cls_head.shard_rollover_date = head.get_shard_rollover_date(); - return cls_rgw_lc_put_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head); + librados::ObjectWriteOperation op; + cls_rgw_lc_put_head(op, cls_head); + + auto& ioctx = *store->getRados()->get_lc_pool_ctx(); + return rgw_rados_operate(dpp, ioctx, oid, &op, y); } std::unique_ptr RadosLifecycle::get_serializer(const std::string& lock_name, -- 2.47.3