rgw: AioThrottle uses RGWSI_RADOS::Obj

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/rgw/rgw_putobj_aio.h b/src/rgw/rgw_putobj_aio.h
index 00526bd88630c4cf3ecdc012786c220dcdefeeb0..73a07cc656bc74c3f09a415768d0bca55b3ade90 100644 (file)
--- a/src/rgw/rgw_putobj_aio.h
+++ b/src/rgw/rgw_putobj_aio.h
@@ -16,12 +16,12 @@
 
 #include <boost/intrusive/list.hpp>
 #include "rgw_common.h"
+#include "services/svc_rados.h" // cant forward declare RGWSI_RADOS::Obj
 
 namespace librados {
 class ObjectReadOperation;
 class ObjectWriteOperation;
 }
-struct rgw_rados_ref;
 
 namespace rgw::putobj {
 
@@ -60,11 +60,13 @@ class Aio {
  public:
   virtual ~Aio() {}
 
-  virtual ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
+  virtual ResultList submit(RGWSI_RADOS::Obj& obj,
+                            const rgw_raw_obj& raw_obj,
                             librados::ObjectReadOperation *op,
                             bufferlist *data, uint64_t cost) = 0;
 
-  virtual ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
+  virtual ResultList submit(RGWSI_RADOS::Obj& obj,
+                            const rgw_raw_obj& raw_obj,
                             librados::ObjectWriteOperation *op,
                             uint64_t cost) = 0;
 

diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc
index 8bd11a695155f7e8e259cbbdb313677fea0fe334..315f36f84bd370f99a2a3455ece71cff9943dca4 100644 (file)
--- a/src/rgw/rgw_putobj_processor.cc
+++ b/src/rgw/rgw_putobj_processor.cc
@@ -71,16 +71,11 @@ static int process_completed(const ResultList& completed, RawObjSet *written)
   return error.value_or(0);
 }
 
-int RadosWriter::set_stripe_obj(rgw_raw_obj&& obj)
+int RadosWriter::set_stripe_obj(rgw_raw_obj&& raw_obj)
 {
-  rgw_rados_ref ref;
-  int r = store->get_raw_obj_ref(obj, &ref);
-  if (r < 0) {
-    return r;
-  }
-  stripe_obj = std::move(obj);
-  stripe_ref = std::move(ref);
-  return 0;
+  stripe_raw = std::move(raw_obj);
+  stripe_obj = store->svc.rados->obj(stripe_raw);
+  return stripe_obj.open();
 }
 
 int RadosWriter::process(bufferlist&& bl, uint64_t offset)
@@ -96,7 +91,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset)
   } else {
     op.write(offset, data);
   }
-  auto c = aio->submit(stripe_ref, stripe_obj, &op, cost);
+  auto c = aio->submit(stripe_obj, stripe_raw, &op, cost);
   return process_completed(c, &written);
 }
 
@@ -108,7 +103,7 @@ int RadosWriter::write_exclusive(const bufferlist& data)
   op.create(true); // exclusive create
   op.write_full(data);
 
-  auto c = aio->submit(stripe_ref, stripe_obj, &op, cost);
+  auto c = aio->submit(stripe_obj, stripe_raw, &op, cost);
   auto d = aio->drain();
   c.splice(c.end(), d);
   return process_completed(c, &written);

diff --git a/src/rgw/rgw_putobj_processor.h b/src/rgw/rgw_putobj_processor.h
index c7342833b1abfbb220fc0c7b26278ca2f5ea42c8..65c26f8e3be3af131126d014856a98e3ef55055f 100644 (file)
--- a/src/rgw/rgw_putobj_processor.h
+++ b/src/rgw/rgw_putobj_processor.h
@@ -18,6 +18,7 @@
 
 #include "rgw_putobj.h"
 #include "rgw_rados.h"
+#include "services/svc_rados.h"
 
 namespace rgw::putobj {
 
@@ -76,8 +77,8 @@ class RadosWriter : public DataProcessor {
   const RGWBucketInfo& bucket_info;
   RGWObjectCtx& obj_ctx;
   const rgw_obj& head_obj;
-  rgw_rados_ref stripe_ref; // current stripe ref
-  rgw_raw_obj stripe_obj; // current stripe object
+  rgw_raw_obj stripe_raw;
+  RGWSI_RADOS::Obj stripe_obj; // current stripe object
   RawObjSet written; // set of written objects for deletion
 
  public:

diff --git a/src/rgw/rgw_putobj_throttle.cc b/src/rgw/rgw_putobj_throttle.cc
index e6bbf471195196114de6b29eadd1cc0da4c18ec2..65878884b46d05babb54238d96cef799acca8245 100644 (file)
--- a/src/rgw/rgw_putobj_throttle.cc
+++ b/src/rgw/rgw_putobj_throttle.cc
@@ -36,12 +36,13 @@ bool AioThrottle::waiter_ready() const
   }
 }
 
-ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
+ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj,
+                               const rgw_raw_obj& raw_obj,
                                librados::ObjectWriteOperation *op,
                                uint64_t cost)
 {
   auto p = std::make_unique<Pending>();
-  p->obj = obj;
+  p->obj = raw_obj;
   p->cost = cost;
 
   if (cost > window) {
@@ -49,7 +50,7 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
     completed.push_back(*p);
   } else {
     get(*p);
-    p->result = ref.ioctx.aio_operate(ref.oid, p->completion, op);
+    p->result = obj.aio_operate(p->completion, op);
     if (p->result < 0) {
       put(*p);
     }
@@ -58,12 +59,13 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
   return std::move(completed);
 }
 
-ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
+ResultList AioThrottle::submit(RGWSI_RADOS::Obj& obj,
+                               const rgw_raw_obj& raw_obj,
                                librados::ObjectReadOperation *op,
                                bufferlist *data, uint64_t cost)
 {
   auto p = std::make_unique<Pending>();
-  p->obj = obj;
+  p->obj = raw_obj;
   p->cost = cost;
 
   if (cost > window) {
@@ -71,7 +73,7 @@ ResultList AioThrottle::submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
     completed.push_back(*p);
   } else {
     get(*p);
-    p->result = ref.ioctx.aio_operate(ref.oid, p->completion, op, data);
+    p->result = obj.aio_operate(p->completion, op, data);
     if (p->result < 0) {
       put(*p);
     }

diff --git a/src/rgw/rgw_putobj_throttle.h b/src/rgw/rgw_putobj_throttle.h
index 9a6b78ca6311ef38f6c8bfa286628c06a855643f..3421231d1507610a4e4174d0c14890d6c78cafdc 100644 (file)
--- a/src/rgw/rgw_putobj_throttle.h
+++ b/src/rgw/rgw_putobj_throttle.h
@@ -16,6 +16,7 @@
 
 #include <memory>
 #include "common/ceph_mutex.h"
+#include "services/svc_rados.h"
 #include "rgw_putobj_aio.h"
 
 namespace librados {
@@ -66,12 +67,13 @@ class AioThrottle : public Aio {
     ceph_assert(completed.empty());
   }
 
-  ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
-                    librados::ObjectReadOperation *op, bufferlist *data,
-                    uint64_t cost) override;
+  ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj,
+                    librados::ObjectReadOperation *op,
+                    bufferlist *data, uint64_t cost) override;
 
-  ResultList submit(rgw_rados_ref& ref, const rgw_raw_obj& obj,
-                    librados::ObjectWriteOperation *op, uint64_t cost) override;
+  ResultList submit(RGWSI_RADOS::Obj& obj, const rgw_raw_obj& raw_obj,
+                    librados::ObjectWriteOperation *op,
+                    uint64_t cost) override;
 
   ResultList poll() override;
 

diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc
index ea4214a5339eb5699354f6043bb652d0e1cc1ea1..322e58ce8154da96d349bd3cf05397c1bfc116b6 100644 (file)
--- a/src/test/rgw/test_rgw_throttle.cc
+++ b/src/test/rgw/test_rgw_throttle.cc
@@ -23,46 +23,35 @@ struct RadosEnv : public ::testing::Environment {
  public:
   static constexpr auto poolname = "ceph_test_rgw_throttle";
 
-  static librados::Rados rados;
-  static librados::IoCtx io;
+  static std::optional<RGWSI_RADOS> rados;
 
   void SetUp() override {
-    ASSERT_EQ(0, rados.init_with_context(g_ceph_context));
-    ASSERT_EQ(0, rados.connect());
-    // open/create test pool
-    int r = rados.ioctx_create(poolname, io);
-    if (r == -ENOENT) {
-      r = rados.pool_create(poolname);
-      if (r == -EEXIST) {
-        r = 0;
-      } else if (r == 0) {
-        r = rados.ioctx_create(poolname, io);
-      }
-    }
+    rados.emplace(g_ceph_context);
+    ASSERT_EQ(0, rados->start());
+    int r = rados->pool({poolname}).create();
+    if (r == -EEXIST)
+      r = 0;
     ASSERT_EQ(0, r);
   }
   void TearDown() {
-    rados.shutdown();
+    rados.reset();
   }
 };
-librados::Rados RadosEnv::rados;
-librados::IoCtx RadosEnv::io;
+std::optional<RGWSI_RADOS> RadosEnv::rados;
 
 auto *const rados_env = ::testing::AddGlobalTestEnvironment(new RadosEnv);
 
 // test fixture for global setup/teardown
 class RadosFixture : public ::testing::Test {
  protected:
-  librados::IoCtx& io;
-
-  rgw_raw_obj make_obj(const std::string& oid) {
+  rgw_raw_obj make_raw_obj(const std::string& oid) {
     return {{RadosEnv::poolname}, oid};
   }
-  rgw_rados_ref make_ref(const rgw_raw_obj& obj) {
-    return {obj.pool, obj.oid, "", io};
+  RGWSI_RADOS::Obj make_obj(const rgw_raw_obj& raw) {
+    auto obj = RadosEnv::rados->obj(raw);
+    ceph_assert_always(0 == obj.open());
+    return obj;
   }
- public:
-  RadosFixture() : io(RadosEnv::io) {}
 };
 
 using PutObj_Throttle = RadosFixture;
@@ -79,20 +68,20 @@ std::ostream& operator<<(std::ostream& out, const Result& r) {
 TEST_F(PutObj_Throttle, NoThrottleUpToMax)
 {
   AioThrottle throttle(4);
-  auto obj = make_obj(__PRETTY_FUNCTION__);
-  auto ref = make_ref(obj);
+  auto raw = make_raw_obj(__PRETTY_FUNCTION__);
+  auto obj = make_obj(raw);
   {
     librados::ObjectWriteOperation op1;
-    auto c1 = throttle.submit(ref, obj, &op1, 1);
+    auto c1 = throttle.submit(obj, raw, &op1, 1);
     EXPECT_TRUE(c1.empty());
     librados::ObjectWriteOperation op2;
-    auto c2 = throttle.submit(ref, obj, &op2, 1);
+    auto c2 = throttle.submit(obj, raw, &op2, 1);
     EXPECT_TRUE(c2.empty());
     librados::ObjectWriteOperation op3;
-    auto c3 = throttle.submit(ref, obj, &op3, 1);
+    auto c3 = throttle.submit(obj, raw, &op3, 1);
     EXPECT_TRUE(c3.empty());
     librados::ObjectWriteOperation op4;
-    auto c4 = throttle.submit(ref, obj, &op4, 1);
+    auto c4 = throttle.submit(obj, raw, &op4, 1);
     EXPECT_TRUE(c4.empty());
     // no completions because no ops had to wait
     auto c5 = throttle.poll();
@@ -100,20 +89,20 @@ TEST_F(PutObj_Throttle, NoThrottleUpToMax)
   auto completions = throttle.drain();
   ASSERT_EQ(4u, completions.size());
   for (auto& c : completions) {
-    EXPECT_EQ(Result({obj, -EINVAL}), c);
+    EXPECT_EQ(Result({raw, -EINVAL}), c);
   }
 }
 
 TEST_F(PutObj_Throttle, CostOverWindow)
 {
   AioThrottle throttle(4);
-  auto obj = make_obj(__PRETTY_FUNCTION__);
-  auto ref = make_ref(obj);
+  auto raw = make_raw_obj(__PRETTY_FUNCTION__);
+  auto obj = make_obj(raw);
 
   librados::ObjectWriteOperation op;
-  auto c = throttle.submit(ref, obj, &op, 8);
+  auto c = throttle.submit(obj, raw, &op, 8);
   ASSERT_EQ(1u, c.size());
-  EXPECT_EQ(Result({obj, -EDEADLK}), c.front());
+  EXPECT_EQ(Result({raw, -EDEADLK}), c.front());
 }
 
 TEST_F(PutObj_Throttle, AioThrottleOverMax)
@@ -121,8 +110,8 @@ TEST_F(PutObj_Throttle, AioThrottleOverMax)
   constexpr uint64_t window = 4;
   AioThrottle throttle(window);
 
-  auto obj = make_obj(__PRETTY_FUNCTION__);
-  auto ref = make_ref(obj);
+  auto raw = make_raw_obj(__PRETTY_FUNCTION__);
+  auto obj = make_obj(raw);
 
   // issue 32 writes, and verify that max_outstanding <= window
   constexpr uint64_t total = 32;
@@ -131,7 +120,7 @@ TEST_F(PutObj_Throttle, AioThrottleOverMax)
 
   for (uint64_t i = 0; i < total; i++) {
     librados::ObjectWriteOperation op;
-    auto c = throttle.submit(ref, obj, &op, 1);
+    auto c = throttle.submit(obj, raw, &op, 1);
     outstanding++;
     outstanding -= c.size();
     if (max_outstanding < outstanding) {