From 85fb04416ea1f31389498ea51685db00af7251fd Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Sat, 17 Nov 2018 11:38:02 -0500 Subject: [PATCH] rgw: move aio throttles to namespace rgw they're useful outside of putobj Signed-off-by: Casey Bodley --- src/rgw/CMakeLists.txt | 2 +- src/rgw/{rgw_putobj_aio.h => rgw_aio.h} | 36 +++++++++---------- ...putobj_throttle.cc => rgw_aio_throttle.cc} | 28 +++++++-------- ...w_putobj_throttle.h => rgw_aio_throttle.h} | 28 +++++++-------- src/rgw/rgw_file.h | 4 +-- src/rgw/rgw_op.cc | 8 ++--- src/rgw/rgw_putobj_processor.cc | 4 +-- src/rgw/rgw_putobj_processor.h | 10 ++++-- src/rgw/rgw_rados.cc | 6 ++-- src/test/rgw/test_rgw_throttle.cc | 22 ++++++------ 10 files changed, 76 insertions(+), 72 deletions(-) rename src/rgw/{rgw_putobj_aio.h => rgw_aio.h} (65%) rename src/rgw/{rgw_putobj_throttle.cc => rgw_aio_throttle.cc} (81%) rename src/rgw/{rgw_putobj_throttle.h => rgw_aio_throttle.h} (73%) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 99add82c707..502ab0891ac 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -53,6 +53,7 @@ set(librgw_common_srcs rgw_acl.cc rgw_acl_s3.cc rgw_acl_swift.cc + rgw_aio_throttle.cc rgw_auth.cc rgw_auth_s3.cc rgw_basic_types.cc @@ -97,7 +98,6 @@ set(librgw_common_srcs rgw_policy_s3.cc rgw_putobj.cc rgw_putobj_processor.cc - rgw_putobj_throttle.cc rgw_quota.cc rgw_rados.cc rgw_resolve.cc diff --git a/src/rgw/rgw_putobj_aio.h b/src/rgw/rgw_aio.h similarity index 65% rename from src/rgw/rgw_putobj_aio.h rename to src/rgw/rgw_aio.h index 73a07cc656b..25f828a9dc4 100644 --- a/src/rgw/rgw_putobj_aio.h +++ b/src/rgw/rgw_aio.h @@ -23,14 +23,14 @@ class ObjectReadOperation; class ObjectWriteOperation; } -namespace rgw::putobj { +namespace rgw { -struct Result { +struct AioResult { rgw_raw_obj obj; int result = 0; }; -struct ResultEntry : Result, boost::intrusive::list_base_hook<> { - virtual ~ResultEntry() {} +struct AioResultEntry : AioResult, boost::intrusive::list_base_hook<> { + virtual ~AioResultEntry() {} }; // a list of polymorphic entries that frees them on destruction template @@ -42,10 +42,10 @@ struct OwningList : boost::intrusive::list { OwningList(const OwningList&) = delete; OwningList& operator=(const OwningList&) = delete; }; -using ResultList = OwningList; +using AioResultList = OwningList; // returns the first error code or 0 if all succeeded -inline int check_for_errors(const ResultList& results) { +inline int check_for_errors(const AioResultList& results) { for (auto& e : results) { if (e.result < 0) { return e.result; @@ -60,24 +60,24 @@ class Aio { public: virtual ~Aio() {} - virtual ResultList submit(RGWSI_RADOS::Obj& obj, - const rgw_raw_obj& raw_obj, - librados::ObjectReadOperation *op, - bufferlist *data, uint64_t cost) = 0; + virtual AioResultList submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, + librados::ObjectReadOperation *op, + bufferlist *data, uint64_t cost) = 0; - virtual ResultList submit(RGWSI_RADOS::Obj& obj, - const rgw_raw_obj& raw_obj, - librados::ObjectWriteOperation *op, - uint64_t cost) = 0; + virtual AioResultList submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, + librados::ObjectWriteOperation *op, + uint64_t cost) = 0; // poll for any ready completions without waiting - virtual ResultList poll() = 0; + virtual AioResultList poll() = 0; // return any ready completions. if there are none, wait for the next - virtual ResultList wait() = 0; + virtual AioResultList wait() = 0; // wait for all outstanding completions and return their results - virtual ResultList drain() = 0; + virtual AioResultList drain() = 0; }; -} // namespace rgw::putobj +} // namespace rgw diff --git a/src/rgw/rgw_putobj_throttle.cc b/src/rgw/rgw_aio_throttle.cc similarity index 81% rename from src/rgw/rgw_putobj_throttle.cc rename to src/rgw/rgw_aio_throttle.cc index 65878884b46..bf8a8a82722 100644 --- a/src/rgw/rgw_putobj_throttle.cc +++ b/src/rgw/rgw_aio_throttle.cc @@ -14,10 +14,10 @@ #include "include/rados/librados.hpp" -#include "rgw_putobj_throttle.h" +#include "rgw_aio_throttle.h" #include "rgw_rados.h" -namespace rgw::putobj { +namespace rgw { void AioThrottle::aio_cb(void *cb, void *arg) { @@ -36,10 +36,10 @@ bool AioThrottle::waiter_ready() const } } -ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, - const rgw_raw_obj& raw_obj, - librados::ObjectWriteOperation *op, - uint64_t cost) +AioResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, + librados::ObjectWriteOperation *op, + uint64_t cost) { auto p = std::make_unique(); p->obj = raw_obj; @@ -59,10 +59,10 @@ ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, return std::move(completed); } -ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, - const rgw_raw_obj& raw_obj, - librados::ObjectReadOperation *op, - bufferlist *data, uint64_t cost) +AioResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, + librados::ObjectReadOperation *op, + bufferlist *data, uint64_t cost) { auto p = std::make_unique(); p->obj = raw_obj; @@ -119,13 +119,13 @@ void AioThrottle::put(Pending& p) } } -ResultList AioThrottle::poll() +AioResultList AioThrottle::poll() { std::unique_lock lock{mutex}; return std::move(completed); } -ResultList AioThrottle::wait() +AioResultList AioThrottle::wait() { std::unique_lock lock{mutex}; if (completed.empty() && !pending.empty()) { @@ -137,7 +137,7 @@ ResultList AioThrottle::wait() return std::move(completed); } -ResultList AioThrottle::drain() +AioResultList AioThrottle::drain() { std::unique_lock lock{mutex}; if (!pending.empty()) { @@ -149,4 +149,4 @@ ResultList AioThrottle::drain() return std::move(completed); } -} // namespace rgw::putobj +} // namespace rgw diff --git a/src/rgw/rgw_putobj_throttle.h b/src/rgw/rgw_aio_throttle.h similarity index 73% rename from src/rgw/rgw_putobj_throttle.h rename to src/rgw/rgw_aio_throttle.h index 3421231d150..1f9c53d4b9b 100644 --- a/src/rgw/rgw_putobj_throttle.h +++ b/src/rgw/rgw_aio_throttle.h @@ -17,13 +17,13 @@ #include #include "common/ceph_mutex.h" #include "services/svc_rados.h" -#include "rgw_putobj_aio.h" +#include "rgw_aio.h" namespace librados { class AioCompletion; } -namespace rgw::putobj { +namespace rgw { // a throttle for aio operations that enforces a maximum window on outstanding // bytes. only supports a single waiter, so all public functions must be called @@ -37,13 +37,13 @@ class AioThrottle : public Aio { bool has_completion() const { return !completed.empty(); } bool is_drained() const { return pending.empty(); } - struct Pending : ResultEntry { + struct Pending : AioResultEntry { AioThrottle *parent = nullptr; uint64_t cost = 0; librados::AioCompletion *completion = nullptr; }; OwningList pending; - ResultList completed; + AioResultList completed; enum class Wait { None, Available, Completion, Drained }; Wait waiter = Wait::None; @@ -67,19 +67,19 @@ class AioThrottle : public Aio { ceph_assert(completed.empty()); } - ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, - librados::ObjectReadOperation *op, - bufferlist *data, uint64_t cost) override; + AioResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, + librados::ObjectReadOperation *op, + bufferlist *data, uint64_t cost) override; - ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, - librados::ObjectWriteOperation *op, - uint64_t cost) override; + AioResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, + librados::ObjectWriteOperation *op, + uint64_t cost) override; - ResultList poll() override; + AioResultList poll() override; - ResultList wait() override; + AioResultList wait() override; - ResultList drain() override; + AioResultList drain() override; }; -} // namespace rgw::putobj +} // namespace rgw diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 8702b0060dc..f91399b6de5 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -35,7 +35,7 @@ #include "rgw_ldap.h" #include "rgw_token.h" #include "rgw_putobj_processor.h" -#include "rgw_putobj_throttle.h" +#include "rgw_aio_throttle.h" #include "rgw_compression.h" @@ -2320,7 +2320,7 @@ public: const std::string& bucket_name; const std::string& obj_name; RGWFileHandle* rgw_fh; - std::optional aio; + std::optional aio; std::optional processor; rgw::putobj::DataProcessor* filter; boost::optional compressor; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 552e3894ff5..2f8044981fb 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -30,6 +30,7 @@ #include "rgw_acl.h" #include "rgw_acl_s3.h" #include "rgw_acl_swift.h" +#include "rgw_aio_throttle.h" #include "rgw_user.h" #include "rgw_bucket.h" #include "rgw_log.h" @@ -45,7 +46,6 @@ #include "rgw_role.h" #include "rgw_tag_s3.h" #include "rgw_putobj_processor.h" -#include "rgw_putobj_throttle.h" #include "services/svc_zone.h" #include "services/svc_quota.h" @@ -3468,8 +3468,8 @@ void RGWPutObj::execute() } // create the object processor + rgw::AioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); using namespace rgw::putobj; - AioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); constexpr auto max_processor_size = std::max(sizeof(MultipartObjectProcessor), sizeof(AtomicObjectProcessor)); ceph::static_ptr processor; @@ -3806,8 +3806,8 @@ void RGWPostObj::execute() store->gen_rand_obj_instance_name(&obj); } + rgw::AioThrottle aio(s->cct->_conf->rgw_put_obj_min_window_size); using namespace rgw::putobj; - AioThrottle aio(s->cct->_conf->rgw_put_obj_min_window_size); AtomicObjectProcessor processor(&aio, store, s->bucket_info, s->bucket_owner.get_id(), *static_cast(s->obj_ctx), @@ -6579,8 +6579,8 @@ int RGWBulkUploadOp::handle_file(const boost::string_ref path, store->gen_rand_obj_instance_name(&obj); } + rgw::AioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); using namespace rgw::putobj; - AioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); AtomicObjectProcessor processor(&aio, store, binfo, bowner.get_id(), obj_ctx, obj, 0, s->req_id); diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 315f36f84bd..4dbb2599fca 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -12,7 +12,7 @@ * */ -#include "rgw_putobj_aio.h" +#include "rgw_aio.h" #include "rgw_putobj_processor.h" #include "rgw_multi.h" #include "services/svc_sys_obj.h" @@ -58,7 +58,7 @@ int HeadObjectProcessor::process(bufferlist&& data, uint64_t logical_offset) } -static int process_completed(const ResultList& completed, RawObjSet *written) +static int process_completed(const AioResultList& completed, RawObjSet *written) { std::optional error; for (auto& r : completed) { diff --git a/src/rgw/rgw_putobj_processor.h b/src/rgw/rgw_putobj_processor.h index 65c26f8e3be..4aca1a3225d 100644 --- a/src/rgw/rgw_putobj_processor.h +++ b/src/rgw/rgw_putobj_processor.h @@ -20,7 +20,11 @@ #include "rgw_rados.h" #include "services/svc_rados.h" -namespace rgw::putobj { +namespace rgw { + +class Aio; + +namespace putobj { // a data consumer that writes an object in a bucket class ObjectProcessor : public DataProcessor { @@ -67,7 +71,6 @@ class HeadObjectProcessor : public ObjectProcessor { }; -class Aio; using RawObjSet = std::set; // a data sink that writes to rados objects and deletes them on cancelation @@ -210,4 +213,5 @@ class MultipartObjectProcessor : public ManifestObjectProcessor { rgw_zone_set *zones_trace, bool *canceled) override; }; -} // namespace rgw::putobj +} // namespace putobj +} // namespace rgw diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 93ce2247ee4..080195b6597 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -24,12 +24,12 @@ #include "rgw_cache.h" #include "rgw_acl.h" #include "rgw_acl_s3.h" /* for dumping s3policy in debug log */ +#include "rgw_aio_throttle.h" #include "rgw_bucket.h" #include "rgw_rest_conn.h" #include "rgw_cr_rados.h" #include "rgw_cr_rest.h" #include "rgw_putobj_processor.h" -#include "rgw_putobj_throttle.h" #include "cls/rgw/cls_rgw_ops.h" #include "cls/rgw/cls_rgw_client.h" @@ -4183,8 +4183,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, obj_time_weight set_mtime_weight; set_mtime_weight.high_precision = high_precision_time; + rgw::AioThrottle aio(cct->_conf->rgw_put_obj_min_window_size); using namespace rgw::putobj; - AioThrottle aio(cct->_conf->rgw_put_obj_min_window_size); AtomicObjectProcessor processor(&aio, this, dest_bucket_info, user_id, obj_ctx, dest_obj, olh_epoch, tag); int ret = processor.prepare(); @@ -4724,8 +4724,8 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx, string tag; append_rand_alpha(cct, tag, tag, 32); + rgw::AioThrottle aio(cct->_conf->rgw_put_obj_min_window_size); using namespace rgw::putobj; - AioThrottle aio(cct->_conf->rgw_put_obj_min_window_size); AtomicObjectProcessor processor(&aio, this, dest_bucket_info, dest_bucket_info.owner, obj_ctx, dest_obj, olh_epoch, tag); diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc index 322e58ce815..37132816c77 100644 --- a/src/test/rgw/test_rgw_throttle.cc +++ b/src/test/rgw/test_rgw_throttle.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_putobj_throttle.h" +#include "rgw/rgw_aio_throttle.h" #include "rgw/rgw_rados.h" #include "include/rados/librados.hpp" @@ -54,18 +54,18 @@ class RadosFixture : public ::testing::Test { } }; -using PutObj_Throttle = RadosFixture; +using Aio_Throttle = RadosFixture; -namespace rgw::putobj { +namespace rgw { -inline bool operator==(const Result& lhs, const Result& rhs) { +inline bool operator==(const AioResult& lhs, const AioResult& rhs) { return lhs.obj == rhs.obj && lhs.result == rhs.result; } -std::ostream& operator<<(std::ostream& out, const Result& r) { +std::ostream& operator<<(std::ostream& out, const AioResult& r) { return out << "{r=" << r.result << " obj='" << r.obj << "'"; } -TEST_F(PutObj_Throttle, NoThrottleUpToMax) +TEST_F(Aio_Throttle, NoThrottleUpToMax) { AioThrottle throttle(4); auto raw = make_raw_obj(__PRETTY_FUNCTION__); @@ -89,11 +89,11 @@ TEST_F(PutObj_Throttle, NoThrottleUpToMax) auto completions = throttle.drain(); ASSERT_EQ(4u, completions.size()); for (auto& c : completions) { - EXPECT_EQ(Result({raw, -EINVAL}), c); + EXPECT_EQ(AioResult({raw, -EINVAL}), c); } } -TEST_F(PutObj_Throttle, CostOverWindow) +TEST_F(Aio_Throttle, CostOverWindow) { AioThrottle throttle(4); auto raw = make_raw_obj(__PRETTY_FUNCTION__); @@ -102,10 +102,10 @@ TEST_F(PutObj_Throttle, CostOverWindow) librados::ObjectWriteOperation op; auto c = throttle.submit(obj, raw, &op, 8); ASSERT_EQ(1u, c.size()); - EXPECT_EQ(Result({raw, -EDEADLK}), c.front()); + EXPECT_EQ(AioResult({raw, -EDEADLK}), c.front()); } -TEST_F(PutObj_Throttle, AioThrottleOverMax) +TEST_F(Aio_Throttle, ThrottleOverMax) { constexpr uint64_t window = 4; AioThrottle throttle(window); @@ -133,4 +133,4 @@ TEST_F(PutObj_Throttle, AioThrottleOverMax) EXPECT_EQ(window, max_outstanding); } -} // namespace rgw::putobj +} // namespace rgw -- 2.39.5