From 27664f22c41d0d9397ac7ef19019c64659987df4 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 23 Oct 2018 17:29:54 -0400 Subject: [PATCH] rgw: AioThrottle uses RGWSI_RADOS::Obj Signed-off-by: Casey Bodley --- src/rgw/rgw_putobj_aio.h | 8 ++-- src/rgw/rgw_putobj_processor.cc | 17 +++----- src/rgw/rgw_putobj_processor.h | 5 ++- src/rgw/rgw_putobj_throttle.cc | 14 ++++--- src/rgw/rgw_putobj_throttle.h | 12 +++--- src/test/rgw/test_rgw_throttle.cc | 65 +++++++++++++------------------ 6 files changed, 56 insertions(+), 65 deletions(-) diff --git a/src/rgw/rgw_putobj_aio.h b/src/rgw/rgw_putobj_aio.h index 00526bd8863..73a07cc656b 100644 --- a/src/rgw/rgw_putobj_aio.h +++ b/src/rgw/rgw_putobj_aio.h @@ -16,12 +16,12 @@ #include #include "rgw_common.h" +#include "services/svc_rados.h" // cant forward declare RGWSI_RADOS::Obj namespace librados { class ObjectReadOperation; class ObjectWriteOperation; } -struct rgw_rados_ref; namespace rgw::putobj { @@ -60,11 +60,13 @@ class Aio { public: virtual ~Aio() {} - virtual ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, + virtual ResultList submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, librados::ObjectReadOperation *op, bufferlist *data, uint64_t cost) = 0; - virtual ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, + virtual ResultList submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, librados::ObjectWriteOperation *op, uint64_t cost) = 0; diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 8bd11a69515..315f36f84bd 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -71,16 +71,11 @@ static int process_completed(const ResultList& completed, RawObjSet *written) return error.value_or(0); } -int RadosWriter::set_stripe_obj(rgw_raw_obj&& obj) +int RadosWriter::set_stripe_obj(rgw_raw_obj&& raw_obj) { - rgw_rados_ref ref; - int r = store->get_raw_obj_ref(obj, &ref); - if (r < 0) { - return r; - } - stripe_obj = std::move(obj); - stripe_ref = std::move(ref); - return 0; + stripe_raw = std::move(raw_obj); + stripe_obj = store->svc.rados->obj(stripe_raw); + return stripe_obj.open(); } int RadosWriter::process(bufferlist&& bl, uint64_t offset) @@ -96,7 +91,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset) } else { op.write(offset, data); } - auto c = aio->submit(stripe_ref, stripe_obj, &op, cost); + auto c = aio->submit(stripe_obj, stripe_raw, &op, cost); return process_completed(c, &written); } @@ -108,7 +103,7 @@ int RadosWriter::write_exclusive(const bufferlist& data) op.create(true); // exclusive create op.write_full(data); - auto c = aio->submit(stripe_ref, stripe_obj, &op, cost); + auto c = aio->submit(stripe_obj, stripe_raw, &op, cost); auto d = aio->drain(); c.splice(c.end(), d); return process_completed(c, &written); diff --git a/src/rgw/rgw_putobj_processor.h b/src/rgw/rgw_putobj_processor.h index c7342833b1a..65c26f8e3be 100644 --- a/src/rgw/rgw_putobj_processor.h +++ b/src/rgw/rgw_putobj_processor.h @@ -18,6 +18,7 @@ #include "rgw_putobj.h" #include "rgw_rados.h" +#include "services/svc_rados.h" namespace rgw::putobj { @@ -76,8 +77,8 @@ class RadosWriter : public DataProcessor { const RGWBucketInfo& bucket_info; RGWObjectCtx& obj_ctx; const rgw_obj& head_obj; - rgw_rados_ref stripe_ref; // current stripe ref - rgw_raw_obj stripe_obj; // current stripe object + rgw_raw_obj stripe_raw; + RGWSI_RADOS::Obj stripe_obj; // current stripe object RawObjSet written; // set of written objects for deletion public: diff --git a/src/rgw/rgw_putobj_throttle.cc b/src/rgw/rgw_putobj_throttle.cc index e6bbf471195..65878884b46 100644 --- a/src/rgw/rgw_putobj_throttle.cc +++ b/src/rgw/rgw_putobj_throttle.cc @@ -36,12 +36,13 @@ bool AioThrottle::waiter_ready() const } } -ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, +ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, librados::ObjectWriteOperation *op, uint64_t cost) { auto p = std::make_unique(); - p->obj = obj; + p->obj = raw_obj; p->cost = cost; if (cost > window) { @@ -49,7 +50,7 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, completed.push_back(*p); } else { get(*p); - p->result = ref.ioctx.aio_operate(ref.oid, p->completion, op); + p->result = obj.aio_operate(p->completion, op); if (p->result < 0) { put(*p); } @@ -58,12 +59,13 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, return std::move(completed); } -ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, +ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj, + const rgw_raw_obj& raw_obj, librados::ObjectReadOperation *op, bufferlist *data, uint64_t cost) { auto p = std::make_unique(); - p->obj = obj; + p->obj = raw_obj; p->cost = cost; if (cost > window) { @@ -71,7 +73,7 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, completed.push_back(*p); } else { get(*p); - p->result = ref.ioctx.aio_operate(ref.oid, p->completion, op, data); + p->result = obj.aio_operate(p->completion, op, data); if (p->result < 0) { put(*p); } diff --git a/src/rgw/rgw_putobj_throttle.h b/src/rgw/rgw_putobj_throttle.h index 9a6b78ca631..3421231d150 100644 --- a/src/rgw/rgw_putobj_throttle.h +++ b/src/rgw/rgw_putobj_throttle.h @@ -16,6 +16,7 @@ #include #include "common/ceph_mutex.h" +#include "services/svc_rados.h" #include "rgw_putobj_aio.h" namespace librados { @@ -66,12 +67,13 @@ class AioThrottle : public Aio { ceph_assert(completed.empty()); } - ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, - librados::ObjectReadOperation *op, bufferlist *data, - uint64_t cost) override; + ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, + librados::ObjectReadOperation *op, + bufferlist *data, uint64_t cost) override; - ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj, - librados::ObjectWriteOperation *op, uint64_t cost) override; + ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj, + librados::ObjectWriteOperation *op, + uint64_t cost) override; ResultList poll() override; diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc index ea4214a5339..322e58ce815 100644 --- a/src/test/rgw/test_rgw_throttle.cc +++ b/src/test/rgw/test_rgw_throttle.cc @@ -23,46 +23,35 @@ struct RadosEnv : public ::testing::Environment { public: static constexpr auto poolname = "ceph_test_rgw_throttle"; - static librados::Rados rados; - static librados::IoCtx io; + static std::optional rados; void SetUp() override { - ASSERT_EQ(0, rados.init_with_context(g_ceph_context)); - ASSERT_EQ(0, rados.connect()); - // open/create test pool - int r = rados.ioctx_create(poolname, io); - if (r == -ENOENT) { - r = rados.pool_create(poolname); - if (r == -EEXIST) { - r = 0; - } else if (r == 0) { - r = rados.ioctx_create(poolname, io); - } - } + rados.emplace(g_ceph_context); + ASSERT_EQ(0, rados->start()); + int r = rados->pool({poolname}).create(); + if (r == -EEXIST) + r = 0; ASSERT_EQ(0, r); } void TearDown() { - rados.shutdown(); + rados.reset(); } }; -librados::Rados RadosEnv::rados; -librados::IoCtx RadosEnv::io; +std::optional RadosEnv::rados; auto *const rados_env = ::testing::AddGlobalTestEnvironment(new RadosEnv); // test fixture for global setup/teardown class RadosFixture : public ::testing::Test { protected: - librados::IoCtx& io; - - rgw_raw_obj make_obj(const std::string& oid) { + rgw_raw_obj make_raw_obj(const std::string& oid) { return {{RadosEnv::poolname}, oid}; } - rgw_rados_ref make_ref(const rgw_raw_obj& obj) { - return {obj.pool, obj.oid, "", io}; + RGWSI_RADOS::Obj make_obj(const rgw_raw_obj& raw) { + auto obj = RadosEnv::rados->obj(raw); + ceph_assert_always(0 == obj.open()); + return obj; } - public: - RadosFixture() : io(RadosEnv::io) {} }; using PutObj_Throttle = RadosFixture; @@ -79,20 +68,20 @@ std::ostream& operator<<(std::ostream& out, const Result& r) { TEST_F(PutObj_Throttle, NoThrottleUpToMax) { AioThrottle throttle(4); - auto obj = make_obj(__PRETTY_FUNCTION__); - auto ref = make_ref(obj); + auto raw = make_raw_obj(__PRETTY_FUNCTION__); + auto obj = make_obj(raw); { librados::ObjectWriteOperation op1; - auto c1 = throttle.submit(ref, obj, &op1, 1); + auto c1 = throttle.submit(obj, raw, &op1, 1); EXPECT_TRUE(c1.empty()); librados::ObjectWriteOperation op2; - auto c2 = throttle.submit(ref, obj, &op2, 1); + auto c2 = throttle.submit(obj, raw, &op2, 1); EXPECT_TRUE(c2.empty()); librados::ObjectWriteOperation op3; - auto c3 = throttle.submit(ref, obj, &op3, 1); + auto c3 = throttle.submit(obj, raw, &op3, 1); EXPECT_TRUE(c3.empty()); librados::ObjectWriteOperation op4; - auto c4 = throttle.submit(ref, obj, &op4, 1); + auto c4 = throttle.submit(obj, raw, &op4, 1); EXPECT_TRUE(c4.empty()); // no completions because no ops had to wait auto c5 = throttle.poll(); @@ -100,20 +89,20 @@ TEST_F(PutObj_Throttle, NoThrottleUpToMax) auto completions = throttle.drain(); ASSERT_EQ(4u, completions.size()); for (auto& c : completions) { - EXPECT_EQ(Result({obj, -EINVAL}), c); + EXPECT_EQ(Result({raw, -EINVAL}), c); } } TEST_F(PutObj_Throttle, CostOverWindow) { AioThrottle throttle(4); - auto obj = make_obj(__PRETTY_FUNCTION__); - auto ref = make_ref(obj); + auto raw = make_raw_obj(__PRETTY_FUNCTION__); + auto obj = make_obj(raw); librados::ObjectWriteOperation op; - auto c = throttle.submit(ref, obj, &op, 8); + auto c = throttle.submit(obj, raw, &op, 8); ASSERT_EQ(1u, c.size()); - EXPECT_EQ(Result({obj, -EDEADLK}), c.front()); + EXPECT_EQ(Result({raw, -EDEADLK}), c.front()); } TEST_F(PutObj_Throttle, AioThrottleOverMax) @@ -121,8 +110,8 @@ TEST_F(PutObj_Throttle, AioThrottleOverMax) constexpr uint64_t window = 4; AioThrottle throttle(window); - auto obj = make_obj(__PRETTY_FUNCTION__); - auto ref = make_ref(obj); + auto raw = make_raw_obj(__PRETTY_FUNCTION__); + auto obj = make_obj(raw); // issue 32 writes, and verify that max_outstanding <= window constexpr uint64_t total = 32; @@ -131,7 +120,7 @@ TEST_F(PutObj_Throttle, AioThrottleOverMax) for (uint64_t i = 0; i < total; i++) { librados::ObjectWriteOperation op; - auto c = throttle.submit(ref, obj, &op, 1); + auto c = throttle.submit(obj, raw, &op, 1); outstanding++; outstanding -= c.size(); if (max_outstanding < outstanding) { -- 2.39.5