rgw: various ioctx.operate refactoring changes

- remove unused #include for cls_client headers
- refactor cls_timeindex_trim call in
rgw_object_expirer_core.cc
- add more CLS_CLIENT_HIDE_IOCTX in header files
- move target_compile_definitions to rgw_a target
- make repeat cls helper functions static

Signed-off-by: Ali Maredia <amaredia@redhat.com>

diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h
index 0e701ff919adc50e89c61f8648fb22a2f4dfdc44..059e7750216f62c8e5f8f92edfd769b7f80e1679 100644 (file)
--- a/src/cls/log/cls_log_client.h
+++ b/src/cls/log/cls_log_client.h
@@ -23,8 +23,11 @@ void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
 
 void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,
                   const string& from_marker, const string& to_marker);
+
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, const utime_t& from_time, const utime_t& to_time,
                  const string& from_marker, const string& to_marker);
+#endif
 
 void cls_log_info(librados::ObjectReadOperation& op, cls_log_header *header);
 

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 453f243533489d8a29df257ed9f28dbdc1c24436..fdd73d52a0371497d4fc2382870f75c2df81925c 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -423,7 +423,7 @@ void cls_rgw_bucket_unlink_instance(librados::ObjectWriteOperation& op,
   op.exec(RGW_CLASS, RGW_BUCKET_UNLINK_INSTANCE, in);
 }
 
-void cls_rgw_get_olh_log(librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker, const string& olh_tag, rgw_cls_read_olh_log_ret *ret, int* op_ret)
+void cls_rgw_get_olh_log(librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker, const string& olh_tag, rgw_cls_read_olh_log_ret& log_ret, int& op_ret)
 {
   bufferlist in;
   rgw_cls_read_olh_log_op call;
@@ -431,21 +431,22 @@ void cls_rgw_get_olh_log(librados::ObjectReadOperation& op, const cls_rgw_obj_ke
   call.ver_marker = ver_marker;
   call.olh_tag = olh_tag;
   encode(call, in);
-  op.exec(RGW_CLASS, RGW_BUCKET_READ_OLH_LOG, in, new ClsBucketIndexOpCtx<rgw_cls_read_olh_log_ret>(ret, op_ret));
+  op.exec(RGW_CLASS, RGW_BUCKET_READ_OLH_LOG, in, new ClsBucketIndexOpCtx<rgw_cls_read_olh_log_ret>(&log_ret, &op_ret));
 }
 
-int cls_rgw_get_olh_log(IoCtx& io_ctx, string& oid, librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker,
+int cls_rgw_get_olh_log(IoCtx& io_ctx, string& oid, const cls_rgw_obj_key& olh, uint64_t ver_marker,
                         const string& olh_tag,
-                        rgw_cls_read_olh_log_ret *ret)
+                        rgw_cls_read_olh_log_ret& log_ret)
 {
-  int *op_ret;
-  cls_rgw_get_olh_log(op, olh, ver_marker, olh_tag, ret, op_ret);
+  int op_ret = 0;
+  librados::ObjectReadOperation op;
+  cls_rgw_get_olh_log(op, olh, ver_marker, olh_tag, log_ret, op_ret);
   int r = io_ctx.operate(oid, &op, NULL);
   if (r < 0) {
     return r;
   }
   if (op_ret < 0) {
-    return *op_ret;
+    return op_ret;
   }
 
  return r;

diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 7174a791530ba7a591cba42a8675dd1e5e34fe2a..5085ccdf52ce8d5dfa975b0ae3f6c2cbe5ecc3ba 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -387,7 +387,7 @@ void cls_rgw_bucket_link_olh(librados::ObjectWriteOperation& op,
 void cls_rgw_bucket_unlink_instance(librados::ObjectWriteOperation& op,
                                    const cls_rgw_obj_key& key, const string& op_tag,
                                    const string& olh_tag, uint64_t olh_epoch, bool log_op, rgw_zone_set& zones_trace);
-void cls_rgw_get_olh_log(librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker, const string& olh_tag, rgw_cls_read_olh_log_ret *ret, int* op_ret);
+void cls_rgw_get_olh_log(librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker, const string& olh_tag, rgw_cls_read_olh_log_ret& log_ret, int& op_ret);
 void cls_rgw_trim_olh_log(librados::ObjectWriteOperation& op, const cls_rgw_obj_key& olh, uint64_t ver, const string& olh_tag);
 void cls_rgw_clear_olh(librados::ObjectWriteOperation& op, const cls_rgw_obj_key& olh, const string& olh_tag);
 
@@ -401,8 +401,8 @@ int cls_rgw_bucket_link_olh(librados::IoCtx& io_ctx, const string& oid,
 int cls_rgw_bucket_unlink_instance(librados::IoCtx& io_ctx, const string& oid,
                                    const cls_rgw_obj_key& key, const string& op_tag,
                                    const string& olh_tag, uint64_t olh_epoch, bool log_op, rgw_zone_set& zones_trace);
-int cls_rgw_get_olh_log(librados::IoCtx& io_ctx, string& oid, librados::ObjectReadOperation& op, const cls_rgw_obj_key& olh, uint64_t ver_marker,
-                        const string& olh_tag, rgw_cls_read_olh_log_ret *ret);
+int cls_rgw_get_olh_log(librados::IoCtx& io_ctx, string& oid, const cls_rgw_obj_key& olh, uint64_t ver_marker,
+                        const string& olh_tag, rgw_cls_read_olh_log_ret& log_ret);
 int cls_rgw_clear_olh(librados::IoCtx& io_ctx, string& oid, const cls_rgw_obj_key& olh, const string& olh_tag);
 int cls_rgw_usage_log_trim(librados::IoCtx& io_ctx, const string& oid, const string& user, const string& bucket,
                            uint64_t start_epoch, uint64_t end_epoch);
@@ -566,9 +566,13 @@ void cls_rgw_encode_suggestion(char op, rgw_bucket_dir_entry& dirent, bufferlist
 void cls_rgw_suggest_changes(librados::ObjectWriteOperation& o, bufferlist& updates);
 
 /* usage logging */
+// these overloads which call io_ctx.operate() should not be called in the rgw.
+// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_rgw_usage_log_read(librados::IoCtx& io_ctx, const string& oid, const string& user, const string& bucket,
                            uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, string& read_iter,
                           map<rgw_user_bucket, rgw_usage_log_entry>& usage, bool *is_truncated);
+#endif
 
 void cls_rgw_usage_log_trim(librados::ObjectWriteOperation& op, const string& user, const string& bucket, uint64_t start_epoch, uint64_t end_epoch);
 
@@ -578,13 +582,19 @@ void cls_rgw_usage_log_add(librados::ObjectWriteOperation& op, rgw_usage_log_inf
 /* garbage collection */
 void cls_rgw_gc_set_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, cls_rgw_gc_obj_info& info);
 void cls_rgw_gc_defer_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, const string& tag);
+void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const vector<string>& tags);
 
+// these overloads which call io_ctx.operate() should not be called in the rgw.
+// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only,
                     list<cls_rgw_gc_obj_info>& entries, bool *truncated, string& next_marker);
-
-void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const vector<string>& tags);
+#endif
 
 /* lifecycle */
+// these overloads which call io_ctx.operate() should not be called in the rgw.
+// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_rgw_lc_get_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head);
 int cls_rgw_lc_put_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head);
 int cls_rgw_lc_get_next_entry(librados::IoCtx& io_ctx, const string& oid, string& marker, pair<string, int>& entry);
@@ -595,20 +605,29 @@ int cls_rgw_lc_list(librados::IoCtx& io_ctx, const string& oid,
                     const string& marker,
                     uint32_t max_entries,
                     map<string, int>& entries);
+#endif
 
 /* resharding */
 void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry);
+void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry);
+// these overloads which call io_ctx.operate() should not be called in the rgw.
+// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_rgw_reshard_list(librados::IoCtx& io_ctx, const string& oid, string& marker, uint32_t max,
                          list<cls_rgw_reshard_entry>& entries, bool* is_truncated);
 int cls_rgw_reshard_get(librados::IoCtx& io_ctx, const string& oid, cls_rgw_reshard_entry& entry);
-void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry);
+#endif
 
 /* resharding attribute on bucket index shard headers */
+void cls_rgw_guard_bucket_resharding(librados::ObjectOperation& op, int ret_err);
+// these overloads which call io_ctx.operate() should not be called in the rgw.
+// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_rgw_set_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
                                   const cls_rgw_bucket_instance_entry& entry);
 int cls_rgw_clear_bucket_resharding(librados::IoCtx& io_ctx, const string& oid);
-void cls_rgw_guard_bucket_resharding(librados::ObjectOperation& op, int ret_err);
 int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
                                   cls_rgw_bucket_instance_entry *entry);
+#endif
 
 #endif

diff --git a/src/cls/timeindex/cls_timeindex_client.h b/src/cls/timeindex/cls_timeindex_client.h
index fc57a1600b72ee308c42b441ab03b93616db2c28..49fa5ce4afcc6c1791fdbb10aae25d828f469fa1 100644 (file)
--- a/src/cls/timeindex/cls_timeindex_client.h
+++ b/src/cls/timeindex/cls_timeindex_client.h
@@ -83,6 +83,7 @@ void cls_timeindex_trim(
   const std::string& from_marker = std::string(),
   const std::string& to_marker = std::string());
 
+#ifndef CLS_CLIENT_HIDE_IOCTX
 int cls_timeindex_trim(
   librados::IoCtx& io_ctx,
   const std::string& oid,
@@ -91,3 +92,5 @@ int cls_timeindex_trim(
   const std::string& from_marker = std::string(),
   const std::string& to_marker = std::string());
 #endif
+
+#endif

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index c0f39db9e3848768c04e7e0b3e0aaeee11bb80aa..bca42c4d4df6da1ac54a1c6b3a93da0ccd0526f9 100644 (file)
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -201,6 +201,7 @@ add_library(rgw_a STATIC
 
 add_dependencies(rgw_a civetweb_h)
 
+target_compile_definitions(rgw_a PUBLIC "-DCLS_CLIENT_HIDE_IOCTX")
 target_include_directories(rgw_a PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src")
 target_include_directories(rgw_a SYSTEM PUBLIC "../rapidjson/include")
 
@@ -261,7 +262,6 @@ target_link_libraries(rgw_schedulers
 target_link_libraries(radosgw_a
   PRIVATE ${rgw_libs} rgw_schedulers
   PUBLIC dmclock::dmclock)
-target_compile_definitions(radosgw_a PUBLIC "-DCLS_CLIENT_HIDE_IOCTX")
 if(WITH_RADOSGW_BEAST_FRONTEND AND WITH_RADOSGW_BEAST_OPENSSL)
   # used by rgw_asio_frontend.cc
   target_link_libraries(radosgw_a PRIVATE OpenSSL::SSL)

diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc
index 0e78788649e4f8a873892d5208c1f1b99fef99a5..781c4b808a0ad48f82cabe7866b04bc3c010352a 100644 (file)
--- a/src/rgw/rgw_object_expirer_core.cc
+++ b/src/rgw/rgw_object_expirer_core.cc
@@ -147,6 +147,27 @@ int RGWObjExpStore::objexp_hint_list(const string& oid,
   return 0;
 }
 
+static int cls_timeindex_trim_repeat(rgw_rados_ref ref,
+                                const string& oid,
+                                const utime_t& from_time,
+                                const utime_t& to_time,
+                                const string& from_marker,
+                                const string& to_marker)
+{
+  bool done = false;
+  do {
+    librados::ObjectWriteOperation op;
+    cls_timeindex_trim(op, from_time, to_time, from_marker, to_marker);
+    int r = rgw_rados_operate(ref.pool.ioctx(), oid, &op, null_yield);
+    if (r == -ENODATA)
+      done = true;
+    else if (r < 0)
+      return r;
+  } while (!done);
+
+  return 0;
+}
+
 int RGWObjExpStore::objexp_hint_trim(const string& oid,
                                const ceph::real_time& start_time,
                                const ceph::real_time& end_time,
@@ -160,7 +181,7 @@ int RGWObjExpStore::objexp_hint_trim(const string& oid,
     return r;
   }
   auto& ref = obj.get_ref();
-  int ret = cls_timeindex_trim(ref.pool.ioctx(), ref.obj.oid, utime_t(start_time), utime_t(end_time),
+  int ret = cls_timeindex_trim_repeat(ref, oid, utime_t(start_time), utime_t(end_time),
           from_marker, to_marker);
   if ((ret < 0 ) && (ret != -ENOENT)) {
     return ret;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 749b23a6259615d79d5b62614ef04fa0d19fad06..f3ac0810c32d77c6f29c936e7fb6411d05c63a2f 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -36,10 +36,6 @@
 #include "cls/rgw/cls_rgw_const.h"
 #include "cls/refcount/cls_refcount_client.h"
 #include "cls/version/cls_version_client.h"
-#include "cls/log/cls_log_client.h"
-#include "cls/lock/cls_lock_client.h"
-#include "cls/user/cls_user_client.h"
-#include "cls/otp/cls_otp_client.h"
 #include "osd/osd_types.h"
 
 #include "rgw_tools.h"
@@ -4101,6 +4097,8 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
       const rgw_raw_obj& loc = miter.get_location().get_raw_obj(this);
 
       auto& ioctx = ref.pool.ioctx();
+      ioctx.locator_set_key(loc.loc);
+
       ret = rgw_rados_operate(ioctx, loc.oid, &op, null_yield);
       if (ret < 0) {
         goto done_ret;
@@ -6579,18 +6577,20 @@ int RGWRados::bucket_index_read_olh_log(const RGWBucketInfo& bucket_info, RGWObj
                        ObjectReadOperation op;
                        cls_rgw_guard_bucket_resharding(op, -ERR_BUSY_RESHARDING);
 
-                        rgw_cls_read_olh_log_ret *ret;
-                        int *op_ret;
-                       cls_rgw_get_olh_log(op, key, ver_marker, olh_tag, ret, op_ret); 
+                        rgw_cls_read_olh_log_ret log_ret;
+                        int op_ret = 0;
+                       cls_rgw_get_olh_log(op, key, ver_marker, olh_tag, log_ret, op_ret); 
                         bufferlist outbl;
                         int r =  rgw_rados_operate(ref.pool.ioctx(), ref.obj.oid, &op, &outbl, null_yield);
                         if (r < 0) {
                           return r;
                         }
                         if (op_ret < 0) {
-                          return *op_ret;
+                          return op_ret;
                         }
 
+                        *log = std::move(log_ret.log);
+                        *is_truncated = log_ret.is_truncated;
                         return r;
                      });
   if (ret < 0) {
@@ -8236,7 +8236,7 @@ int RGWRados::cls_obj_usage_log_read(const string& oid, const string& user, cons
   return r;
 }
 
-int RGWRados::cls_rgw_usage_log_trim_repeat(rgw_rados_ref ref, const string& user, const string& bucket, uint64_t start_epoch, uint64_t end_epoch)
+static int cls_rgw_usage_log_trim_repeat(rgw_rados_ref ref, const string& user, const string& bucket, uint64_t start_epoch, uint64_t end_epoch)
 {
   bool done = false;
   do {

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 8b18889fc2230be40523873d93c3601c4336fda3..d15caeecf956d6a6731a37f6e5389bf5187d1038 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1380,7 +1380,6 @@ public:
   int cls_obj_usage_log_read(const string& oid, const string& user, const string& bucket, uint64_t start_epoch,
                              uint64_t end_epoch, uint32_t max_entries, string& read_iter, map<rgw_user_bucket,
                             rgw_usage_log_entry>& usage, bool *is_truncated);
-  int cls_rgw_usage_log_trim_repeat(rgw_rados_ref ref, const string& user, const string& bucket, uint64_t start_epoch, uint64_t end_epoch);
   int cls_obj_usage_log_trim(const string& oid, const string& user, const string& bucket, uint64_t start_epoch,
                              uint64_t end_epoch);
   int cls_obj_usage_log_clear(string& oid);

diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index d447395a246764e9df7b6b1237d49f783dd4fd9d..f0bd4711d2bcc55cb5c0858727f3789ff5823ad7 100644 (file)
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -94,9 +94,8 @@ void index_complete(librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op,
   if (!key.instance.empty()) {
     bufferlist olh_tag;
     olh_tag.append(tag);
-    ObjectWriteOperation op2;
     rgw_zone_set zone_set;
-    ASSERT_EQ(0, cls_rgw_bucket_link_olh(ioctx, op2, oid, key, olh_tag,
+    ASSERT_EQ(0, cls_rgw_bucket_link_olh(ioctx, oid, key, olh_tag,
                                          false, tag, &meta, epoch,
                                          ceph::real_time{}, true, true, zone_set));
   }