]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
rgw/metadata: use raw RGWMetadataHandler for bucket entrypoints
authorCasey Bodley <cbodley@redhat.com>
Sun, 3 Dec 2023 19:44:06 +0000 (14:44 -0500)
committerCasey Bodley <cbodley@redhat.com>
Thu, 12 Sep 2024 20:54:58 +0000 (16:54 -0400)
RGWBucketMetadataHandler inherits from RGWMetadataHandler directly
instead of RGWMetadataHandler_GenericMetaBE

replace RGWSI_Bucket_SObj_Module with the RGWMetadataLister abstraction

RGWSI_Bucket_SObj now depends on RGWSI_MDLog and writes mdlog entries
itself

Signed-off-by: Casey Bodley <cbodley@redhat.com>
src/rgw/driver/rados/rgw_bucket.cc
src/rgw/driver/rados/rgw_bucket.h
src/rgw/driver/rados/rgw_data_sync.cc
src/rgw/driver/rados/rgw_service.cc
src/rgw/driver/rados/rgw_sync_module.cc
src/rgw/driver/rados/rgw_sync_module.h
src/rgw/services/svc_bucket.h
src/rgw/services/svc_bucket_sobj.cc
src/rgw/services/svc_bucket_sobj.h
src/rgw/services/svc_mdlog.cc

index 75db3ea71a09a4751715aff3b52ce7c28cb8c02b..02c401ef9a551d3e9fc8fc3db2cfb40abc528808 100644 (file)
@@ -15,6 +15,7 @@
 
 #include "account.h"
 #include "buckets.h"
+#include "rgw_metadata_lister.h"
 #include "rgw_reshard.h"
 #include "rgw_pubsub.h"
 
@@ -2163,27 +2164,17 @@ void RGWBucketCompleteInfo::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("attrs", attrs, obj);
 }
 
-class RGWBucketMetadataHandler : public RGWBucketMetadataHandlerBase {
-public:
+class RGWBucketMetadataHandler : public RGWMetadataHandler {
+ protected:
   librados::Rados& rados;
-  struct Svc {
-    RGWSI_Bucket *bucket{nullptr};
-  } svc;
+  RGWSI_Bucket* svc_bucket{nullptr};
+  RGWBucketCtl *ctl_bucket{nullptr};
 
-  struct Ctl {
-    RGWBucketCtl *bucket{nullptr};
-  } ctl;
-
-  explicit RGWBucketMetadataHandler(librados::Rados& rados)
-    : rados(rados) {}
-
-  void init(RGWSI_Bucket *bucket_svc,
-            RGWBucketCtl *bucket_ctl) override {
-    base_init(bucket_svc->ctx(),
-              bucket_svc->get_ep_be_handler().get());
-    svc.bucket = bucket_svc;
-    ctl.bucket = bucket_ctl;
-  }
+ public:
+  RGWBucketMetadataHandler(librados::Rados& rados,
+                           RGWSI_Bucket* svc_bucket,
+                           RGWBucketCtl* ctl_bucket)
+    : rados(rados), svc_bucket(svc_bucket), ctl_bucket(ctl_bucket) {}
 
   string get_type() override { return "bucket"; }
 
@@ -2199,146 +2190,77 @@ public:
     return new RGWBucketEntryMetadataObject(be, objv, mtime);
   }
 
-  int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override {
-    RGWObjVersionTracker ot;
-    RGWBucketEntryPoint be;
-
-    real_time mtime;
-    map<string, bufferlist> attrs;
-
-    int ret = svc.bucket->read_bucket_entrypoint_info(entry, &be, &ot, &mtime, &attrs, y, dpp);
-    if (ret < 0)
-      return ret;
-
-    RGWBucketEntryMetadataObject *mdo = new RGWBucketEntryMetadataObject(be, ot.read_version, mtime, std::move(attrs));
-
-    *obj = mdo;
-
-    return 0;
-  }
-
-  int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry,
-             RGWMetadataObject *obj,
-             RGWObjVersionTracker& objv_tracker,
-             optional_yield y,
-             const DoutPrefixProvider *dpp,
-             RGWMDLogSyncType type, bool from_remote_zone) override;
-
-  int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker,
-                optional_yield y, const DoutPrefixProvider *dpp) override {
-    RGWBucketEntryPoint be;
-
-    real_time orig_mtime;
-
-    int ret = svc.bucket->read_bucket_entrypoint_info(entry, &be, &objv_tracker, &orig_mtime, nullptr, y, dpp);
-    if (ret < 0)
-      return ret;
-
-    /*
-     * We're unlinking the bucket but we don't want to update the entrypoint here - we're removing
-     * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal
-     * will incorrectly fail.
-     */
-    ret = ctl.bucket->unlink_bucket(rados, be.owner, be.bucket, y, dpp, false);
-    if (ret < 0) {
-      ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl;
-    }
-
-    ret = svc.bucket->remove_bucket_entrypoint_info(entry, &objv_tracker, y, dpp);
-    if (ret < 0) {
-      ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl;
-    }
-    /* idempotent */
-    return 0;
-  }
-
-  int call(std::function<int(RGWSI_Bucket_EP_Ctx& ctx)> f) {
-    return call(nullopt, f);
-  }
-
-  int call(std::optional<RGWSI_MetaBackend_CtxParams> bectx_params,
-           std::function<int(RGWSI_Bucket_EP_Ctx& ctx)> f) {
-    return be_handler->call(bectx_params, [&](RGWSI_MetaBackend_Handler::Op *op) {
-      RGWSI_Bucket_EP_Ctx ctx(op->ctx());
-      return f(ctx);
-    });
-  }
+  int get(std::string& entry, RGWMetadataObject** obj, optional_yield y,
+          const DoutPrefixProvider *dpp) override;
+  int put(std::string& entry, RGWMetadataObject* obj,
+          RGWObjVersionTracker& objv_tracker,
+          optional_yield y, const DoutPrefixProvider* dpp,
+          RGWMDLogSyncType type, bool from_remote_zone) override;
+  int remove(std::string& entry, RGWObjVersionTracker& objv_tracker,
+             optional_yield y, const DoutPrefixProvider *dpp) override;
+
+  int mutate(const std::string& entry, const ceph::real_time& mtime,
+             RGWObjVersionTracker* objv_tracker, optional_yield y,
+             const DoutPrefixProvider* dpp, RGWMDLogStatus op_type,
+             std::function<int()> f) override;
+
+  int list_keys_init(const DoutPrefixProvider* dpp, const std::string& marker,
+                     void** phandle) override;
+  int list_keys_next(const DoutPrefixProvider* dpp, void* handle, int max,
+                     std::list<std::string>& keys, bool* truncated) override;
+  void list_keys_complete(void *handle) override;
+  std::string get_marker(void *handle) override;
 };
 
-class RGWMetadataHandlerPut_Bucket : public RGWMetadataHandlerPut_SObj
+int RGWBucketMetadataHandler::get(std::string& entry, RGWMetadataObject** obj,
+                                  optional_yield y, const DoutPrefixProvider *dpp)
 {
-  RGWBucketMetadataHandler *bhandler;
-  librados::Rados& rados;
-  RGWBucketEntryMetadataObject *obj;
-public:
-  RGWMetadataHandlerPut_Bucket(RGWBucketMetadataHandler *_handler, librados::Rados& rados,
-                               RGWSI_MetaBackend_Handler::Op *op, string& entry,
-                               RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker,
-                              optional_yield y,
-                               RGWMDLogSyncType type, bool from_remote_zone)
-    : RGWMetadataHandlerPut_SObj(_handler, op, entry, _obj, objv_tracker, y, type, from_remote_zone),
-     bhandler(_handler),
-     rados(rados),
-     obj(static_cast<RGWBucketEntryMetadataObject *>(_obj))
-  {}
+  RGWObjVersionTracker ot;
+  RGWBucketEntryPoint be;
+  real_time mtime;
+  map<string, bufferlist> attrs;
 
-  void encode_obj(bufferlist *bl) override {
-    obj->get_ep().encode(*bl);
+  int ret = svc_bucket->read_bucket_entrypoint_info(entry, &be, &ot, &mtime,
+                                                    &attrs, y, dpp);
+  if (ret < 0) {
+    return ret;
   }
 
-  int put_checked(const DoutPrefixProvider *dpp) override;
-  int put_post(const DoutPrefixProvider *dpp) override;
-};
-
-int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry,
-                                     RGWMetadataObject *obj,
-                                     RGWObjVersionTracker& objv_tracker,
-                                    optional_yield y,
-                                     const DoutPrefixProvider *dpp,
-                                     RGWMDLogSyncType type, bool from_remote_zone)
-{
-  RGWMetadataHandlerPut_Bucket put_op(this, rados, op, entry, obj, objv_tracker,
-                                      y, type, from_remote_zone);
-  return do_put_operate(&put_op, dpp);
+  *obj = new RGWBucketEntryMetadataObject(be, ot.read_version, mtime,
+                                          std::move(attrs));
+  return 0;
 }
 
-int RGWMetadataHandlerPut_Bucket::put_checked(const DoutPrefixProvider *dpp)
+int RGWBucketMetadataHandler::put(std::string& entry, RGWMetadataObject* obj,
+                                  RGWObjVersionTracker& objv_tracker,
+                                  optional_yield y, const DoutPrefixProvider* dpp,
+                                  RGWMDLogSyncType type, bool from_remote_zone)
 {
-  RGWBucketEntryMetadataObject *orig_obj = static_cast<RGWBucketEntryMetadataObject *>(old_obj);
-
-  if (orig_obj) {
-    obj->set_pattrs(&orig_obj->get_attrs());
+  std::optional old_be = RGWBucketEntryPoint{};
+  int ret = svc_bucket->read_bucket_entrypoint_info(
+      entry, &*old_be, &objv_tracker, nullptr, nullptr, y, dpp);
+  if (ret == -ENOENT) {
+    old_be = std::nullopt;
+  } else if (ret < 0) {
+    return ret;
   }
 
-  auto& be = obj->get_ep();
-  auto mtime = obj->get_mtime();
-  auto pattrs = obj->get_pattrs();
+  auto* epobj = static_cast<RGWBucketEntryMetadataObject*>(obj);
+  RGWBucketEntryPoint& new_be = epobj->get_ep();
 
-  RGWSI_Bucket_EP_Ctx ctx(op->ctx());
-
-  return bhandler->svc.bucket->store_bucket_entrypoint_info(entry,
-                                                           be,
-                                                           false,
-                                                           mtime,
-                                                           pattrs,
-                                                           &objv_tracker,
-                                                          y,
-                                                           dpp);
-}
-
-int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider *dpp)
-{
-  auto* orig_obj = static_cast<RGWBucketEntryMetadataObject *>(old_obj);
-  auto* old_be = orig_obj ? &orig_obj->get_ep() : nullptr;
-  auto& new_be = obj->get_ep();
+  ret = svc_bucket->store_bucket_entrypoint_info(
+      entry, new_be, false, obj->get_mtime(),
+      obj->get_pattrs(), &objv_tracker, y, dpp);
+  if (ret < 0) {
+    return ret;
+  }
 
-  RGWBucketCtl& ctl = *bhandler->ctl.bucket;
   constexpr bool update_entrypoint = false;
 
   if (old_be && (old_be->owner != new_be.owner || // owner changed
       (old_be->linked && !new_be.linked))) { // linked -> false
-    int ret = ctl.unlink_bucket(rados, old_be->owner, old_be->bucket,
-                                y, dpp, update_entrypoint);
+    int ret = ctl_bucket->unlink_bucket(rados, old_be->owner, old_be->bucket,
+                                        y, dpp, update_entrypoint);
     if (ret < 0) {
       return ret;
     }
@@ -2346,8 +2268,9 @@ int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider *dpp)
 
   if (new_be.linked && (!old_be || !old_be->linked || // linked -> true
       old_be->owner != new_be.owner)) { // owner changed
-    int ret = ctl.link_bucket(rados, new_be.owner, new_be.bucket,
-                              new_be.creation_time, y, dpp, update_entrypoint);
+    int ret = ctl_bucket->link_bucket(rados, new_be.owner, new_be.bucket,
+                                      new_be.creation_time, y, dpp,
+                                      update_entrypoint);
     if (ret < 0) {
       return ret;
     }
@@ -2442,6 +2365,71 @@ int update_bucket_topic_mappings(const DoutPrefixProvider* dpp,
   return ret;
 }
 
+int RGWBucketMetadataHandler::remove(std::string& entry, RGWObjVersionTracker& objv_tracker,
+                                     optional_yield y, const DoutPrefixProvider *dpp)
+{
+  RGWBucketEntryPoint be;
+  int ret = svc_bucket->read_bucket_entrypoint_info(entry, &be, &objv_tracker,
+                                                    nullptr, nullptr, y, dpp);
+  if (ret < 0) {
+    return ret;
+  }
+
+  constexpr bool update_ep = false; // removing anyway
+  ret = ctl_bucket->unlink_bucket(rados, be.owner, be.bucket, y, dpp, update_ep);
+  if (ret < 0) {
+    ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl;
+  }
+
+  ret = svc_bucket->remove_bucket_entrypoint_info(entry, &objv_tracker, y, dpp);
+  if (ret < 0) {
+    ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl;
+  }
+  /* idempotent */
+  return 0;
+}
+
+int RGWBucketMetadataHandler::mutate(const std::string& entry, const ceph::real_time& mtime,
+                                     RGWObjVersionTracker* objv_tracker, optional_yield y,
+                                     const DoutPrefixProvider* dpp, RGWMDLogStatus op_type,
+                                     std::function<int()> f)
+{
+  return -ENOTSUP; // unused
+}
+
+int RGWBucketMetadataHandler::list_keys_init(const DoutPrefixProvider* dpp,
+                                             const std::string& marker,
+                                             void** phandle)
+{
+  std::unique_ptr<RGWMetadataLister> lister;
+  int ret = svc_bucket->create_entrypoint_lister(dpp, marker, lister);
+  if (ret < 0) {
+    return ret;
+  }
+  *phandle = lister.release(); // release ownership
+  return 0;
+}
+
+int RGWBucketMetadataHandler::list_keys_next(const DoutPrefixProvider* dpp,
+                                             void* handle, int max,
+                                             std::list<std::string>& keys,
+                                             bool* truncated)
+{
+  auto lister = static_cast<RGWMetadataLister*>(handle);
+  return lister->get_next(dpp, max, keys, truncated);
+}
+
+void RGWBucketMetadataHandler::list_keys_complete(void *handle)
+{
+  delete static_cast<RGWMetadataLister*>(handle);
+}
+
+std::string RGWBucketMetadataHandler::get_marker(void *handle)
+{
+  auto lister = static_cast<RGWMetadataLister*>(handle);
+  return lister->get_marker();
+}
+
 static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) {
 
    char md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
@@ -2506,13 +2494,16 @@ struct archive_meta_info {
 WRITE_CLASS_ENCODER(archive_meta_info)
 
 class RGWArchiveBucketMetadataHandler : public RGWBucketMetadataHandler {
-public:
-  explicit RGWArchiveBucketMetadataHandler(librados::Rados& rados)
-    : RGWBucketMetadataHandler(rados) {}
-
-  int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker,
-                optional_yield y, const DoutPrefixProvider *dpp) override {
-    auto cct = svc.bucket->ctx();
+ public:
+  RGWArchiveBucketMetadataHandler(librados::Rados& rados,
+                                  RGWSI_Bucket* svc_bucket,
+                                  RGWBucketCtl* ctl_bucket)
+    : RGWBucketMetadataHandler(rados, svc_bucket, ctl_bucket) {}
+
+  int remove(std::string& entry, RGWObjVersionTracker& objv_tracker,
+             optional_yield y, const DoutPrefixProvider *dpp) override
+  {
+    auto cct = svc_bucket->ctx();
 
     ldpp_dout(dpp, 5) << "SKIP: bucket removal is not allowed on archive zone: bucket:" << entry << " ... proceeding to rename" << dendl;
 
@@ -2528,7 +2519,7 @@ public:
 
     RGWBucketEntryPoint be;
     map<string, bufferlist> attrs;
-    int ret = svc.bucket->read_bucket_entrypoint_info(entry, &be, &objv_tracker, &mtime, &attrs, y, dpp);
+    int ret = svc_bucket->read_bucket_entrypoint_info(entry, &be, &objv_tracker, &mtime, &attrs, y, dpp);
     if (ret < 0) {
         return ret;
     }
@@ -2541,7 +2532,7 @@ public:
     ceph::real_time orig_mtime;
     RGWBucketInfo old_bi;
 
-    ret = ctl.bucket->read_bucket_instance_info(be.bucket, &old_bi, y, dpp, RGWBucketCtl::BucketInstance::GetParams()
+    ret = ctl_bucket->read_bucket_instance_info(be.bucket, &old_bi, y, dpp, RGWBucketCtl::BucketInstance::GetParams()
                                                                     .set_mtime(&orig_mtime)
                                                                     .set_attrs(&attrs_m));
     if (ret < 0) {
@@ -2550,7 +2541,7 @@ public:
 
     archive_meta_info ami;
 
-    if (!ami.from_attrs(svc.bucket->ctx(), attrs_m)) {
+    if (!ami.from_attrs(svc_bucket->ctx(), attrs_m)) {
       ami.orig_bucket = old_bi.bucket;
       ami.store_in_attrs(attrs_m);
     }
@@ -2577,7 +2568,7 @@ public:
 
     new_be.bucket.name = new_bucket_name;
 
-    ret = ctl.bucket->store_bucket_instance_info(new_be.bucket, new_bi, y, dpp, RGWBucketCtl::BucketInstance::PutParams()
+    ret = ctl_bucket->store_bucket_instance_info(new_be.bucket, new_bi, y, dpp, RGWBucketCtl::BucketInstance::PutParams()
                                                                     .set_exclusive(false)
                                                                     .set_mtime(orig_mtime)
                                                                     .set_attrs(&attrs_m)
@@ -2592,7 +2583,7 @@ public:
     RGWObjVersionTracker ot;
     ot.generate_new_write_ver(cct);
 
-    ret = svc.bucket->store_bucket_entrypoint_info(RGWSI_Bucket::get_entrypoint_meta_key(new_be.bucket),
+    ret = svc_bucket->store_bucket_entrypoint_info(RGWSI_Bucket::get_entrypoint_meta_key(new_be.bucket),
                                                    new_be, true, mtime, &attrs, nullptr, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be.bucket << " ret=" << ret << dendl;
@@ -2601,7 +2592,7 @@ public:
 
     /* link new bucket */
 
-    ret = ctl.bucket->link_bucket(rados, new_be.owner, new_be.bucket, new_be.creation_time, y, dpp, false);
+    ret = ctl_bucket->link_bucket(rados, new_be.owner, new_be.bucket, new_be.creation_time, y, dpp, false);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "ERROR: failed to link new bucket for bucket=" << new_be.bucket << " ret=" << ret << dendl;
       return ret;
@@ -2609,7 +2600,7 @@ public:
 
     /* clean up old stuff */
 
-    ret = ctl.bucket->unlink_bucket(rados, be.owner, entry_bucket, y, dpp, false);
+    ret = ctl_bucket->unlink_bucket(rados, be.owner, entry_bucket, y, dpp, false);
     if (ret < 0) {
         ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl;
     }
@@ -2619,7 +2610,7 @@ public:
     // whether it was a newly created bucket entrypoint ...  in which case we
     // should ignore the error and move forward, or whether it is a higher version
     // of the same bucket instance ... in which we should retry
-    ret = svc.bucket->remove_bucket_entrypoint_info(RGWSI_Bucket::get_entrypoint_meta_key(be.bucket),
+    ret = svc_bucket->remove_bucket_entrypoint_info(RGWSI_Bucket::get_entrypoint_meta_key(be.bucket),
                                                     &objv_tracker,
                                                     y,
                                                     dpp);
@@ -2628,7 +2619,7 @@ public:
       return ret;
     }
 
-    ret = ctl.bucket->remove_bucket_instance_info(be.bucket, old_bi, y, dpp);
+    ret = ctl_bucket->remove_bucket_instance_info(be.bucket, old_bi, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl;
     }
@@ -2639,15 +2630,15 @@ public:
     return 0;
   }
 
-  int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry,
-             RGWMetadataObject *obj,
-             RGWObjVersionTracker& objv_tracker,
-             optional_yield y, const DoutPrefixProvider *dpp,
-             RGWMDLogSyncType type, bool from_remote_zone) override {
+  int put(std::string& entry, RGWMetadataObject* obj,
+          RGWObjVersionTracker& objv_tracker,
+          optional_yield y, const DoutPrefixProvider* dpp,
+          RGWMDLogSyncType type, bool from_remote_zone) override
+  {
     if (entry.find("-deleted-") != string::npos) {
       RGWObjVersionTracker ot;
       RGWMetadataObject *robj;
-      int ret = do_get(op, entry, &robj, y, dpp);
+      int ret = get(entry, &robj, y, dpp);
       if (ret != -ENOENT) {
         if (ret < 0) {
           return ret;
@@ -2655,17 +2646,16 @@ public:
         ot.read_version = robj->get_version();
         delete robj;
 
-        ret = do_remove(op, entry, ot, y, dpp);
+        ret = remove(entry, ot, y, dpp);
         if (ret < 0) {
           return ret;
         }
       }
     }
 
-    return RGWBucketMetadataHandler::do_put(op, entry, obj,
-                                            objv_tracker, y, dpp, type, from_remote_zone);
+    return RGWBucketMetadataHandler::put(entry, obj, objv_tracker, y,
+                                         dpp, type, from_remote_zone);
   }
-
 };
 
 class RGWBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandlerBase {
@@ -2978,17 +2968,14 @@ RGWBucketCtl::RGWBucketCtl(RGWSI_Zone *zone_svc,
 }
 
 void RGWBucketCtl::init(RGWUserCtl *user_ctl,
-                        RGWBucketMetadataHandler *_bm_handler,
                         RGWBucketInstanceMetadataHandler *_bmi_handler,
                         RGWDataChangesLog *datalog,
                         const DoutPrefixProvider *dpp)
 {
   ctl.user = user_ctl;
 
-  bm_handler = _bm_handler;
   bmi_handler = _bmi_handler;
 
-  bucket_be_handler = bm_handler->get_be_handler();
   bi_be_handler = bmi_handler->get_be_handler();
 
   datalog->set_bucket_filter(
@@ -3460,9 +3447,13 @@ int RGWBucketCtl::bucket_imports_data(const rgw_bucket& bucket,
   return handler->bucket_imports_data();
 }
 
-RGWBucketMetadataHandlerBase* RGWBucketMetaHandlerAllocator::alloc(librados::Rados& rados)
+auto create_bucket_metadata_handler(librados::Rados& rados,
+                                    RGWSI_Bucket* svc_bucket,
+                                    RGWBucketCtl* ctl_bucket)
+    -> std::unique_ptr<RGWMetadataHandler>
 {
-  return new RGWBucketMetadataHandler(rados);
+  return std::make_unique<RGWBucketMetadataHandler>(
+      rados, svc_bucket, ctl_bucket);
 }
 
 RGWBucketInstanceMetadataHandlerBase* RGWBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Driver* driver)
@@ -3470,9 +3461,13 @@ RGWBucketInstanceMetadataHandlerBase* RGWBucketInstanceMetaHandlerAllocator::all
   return new RGWBucketInstanceMetadataHandler(driver);
 }
 
-RGWBucketMetadataHandlerBase* RGWArchiveBucketMetaHandlerAllocator::alloc(librados::Rados& rados)
+auto create_archive_bucket_metadata_handler(librados::Rados& rados,
+                                            RGWSI_Bucket* svc_bucket,
+                                            RGWBucketCtl* ctl_bucket)
+    -> std::unique_ptr<RGWMetadataHandler>
 {
-  return new RGWArchiveBucketMetadataHandler(rados);
+  return std::make_unique<RGWArchiveBucketMetadataHandler>(
+      rados, svc_bucket, ctl_bucket);
 }
 
 RGWBucketInstanceMetadataHandlerBase* RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Driver* driver)
index a84fb870ea46dd77c1b2e6d7cb06265d3af43a52..ffc46dfc4b23cca0af8d0dec8a12991cf78842c2 100644 (file)
@@ -171,14 +171,6 @@ public:
 };
 WRITE_CLASS_ENCODER(RGWUserBuckets)
 
-class RGWBucketMetadataHandlerBase : public RGWMetadataHandler_GenericMetaBE {
-public:
-  virtual ~RGWBucketMetadataHandlerBase() {}
-  virtual void init(RGWSI_Bucket *bucket_svc,
-                    RGWBucketCtl *bucket_ctl) = 0;
-
-};
-
 class RGWBucketInstanceMetadataHandlerBase : public RGWMetadataHandler_GenericMetaBE {
 public:
   virtual ~RGWBucketInstanceMetadataHandlerBase() {}
@@ -187,20 +179,22 @@ public:
                     RGWSI_BucketIndex *bi_svc) = 0;
 };
 
-class RGWBucketMetaHandlerAllocator {
-public:
-  static RGWBucketMetadataHandlerBase *alloc(librados::Rados& rados);
-};
+// bucket entrypoint metadata handler factory
+auto create_bucket_metadata_handler(librados::Rados& rados,
+                                    RGWSI_Bucket* svc_bucket,
+                                    RGWBucketCtl* ctl_bucket)
+    -> std::unique_ptr<RGWMetadataHandler>;
 
 class RGWBucketInstanceMetaHandlerAllocator {
 public:
   static RGWBucketInstanceMetadataHandlerBase *alloc(rgw::sal::Driver* driver);
 };
 
-class RGWArchiveBucketMetaHandlerAllocator {
-public:
-  static RGWBucketMetadataHandlerBase *alloc(librados::Rados& rados);
-};
+// archive bucket entrypoint metadata handler factory
+auto create_archive_bucket_metadata_handler(librados::Rados& rados,
+                                            RGWSI_Bucket* svc_bucket,
+                                            RGWBucketCtl* ctl_bucket)
+    -> std::unique_ptr<RGWMetadataHandler>;
 
 class RGWArchiveBucketInstanceMetaHandlerAllocator {
 public:
@@ -441,10 +435,8 @@ class RGWBucketCtl {
     RGWUserCtl *user{nullptr};
   } ctl;
 
-  RGWBucketMetadataHandler *bm_handler;
   RGWBucketInstanceMetadataHandler *bmi_handler;
 
-  RGWSI_Bucket_BE_Handler bucket_be_handler; /* bucket backend handler */
   RGWSI_BucketInstance_BE_Handler bi_be_handler; /* bucket instance backend handler */
 
 public:
@@ -455,7 +447,6 @@ public:
                RGWSI_User* user_svc);
 
   void init(RGWUserCtl *user_ctl,
-            RGWBucketMetadataHandler *_bm_handler,
             RGWBucketInstanceMetadataHandler *_bmi_handler,
             RGWDataChangesLog *datalog,
             const DoutPrefixProvider *dpp);
index 58f5cd0690c46d9bbdb0981d4a3a9f3883941d02..248b45845d406f032ce052d40fdc8c2402f71b6d 100644 (file)
@@ -3120,8 +3120,11 @@ public:
   RGWDataSyncModule *get_data_handler() override {
     return &data_handler;
   }
-  RGWMetadataHandler *alloc_bucket_meta_handler(librados::Rados& rados) override {
-    return RGWArchiveBucketMetaHandlerAllocator::alloc(rados);
+  auto alloc_bucket_meta_handler(librados::Rados& rados,
+                                 RGWSI_Bucket* svc_bucket,
+                                 RGWBucketCtl* ctl_bucket)
+      -> std::unique_ptr<RGWMetadataHandler> override {
+    return create_archive_bucket_metadata_handler(rados, svc_bucket, ctl_bucket);
   }
   RGWBucketInstanceMetadataHandlerBase *alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver) override {
     return RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(driver);
index a21c054031e9c041379938de57d5bf9a829f8c97..ba1ebc0626fb554619a8a9d3c44a6e4bd703a4d9 100644 (file)
@@ -93,7 +93,7 @@ int RGWServices_Def::init(CephContext *cct,
                 bilog_rados.get(), datalog_rados.get());
   bilog_rados->init(bi_rados.get());
   bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(),
-                    bi_rados.get(), meta.get(), meta_be_sobj.get(),
+                    bi_rados.get(), meta.get(), mdlog.get(), meta_be_sobj.get(),
                     sync_modules.get(), bucket_sync_sobj.get());
   bucket_sync_sobj->init(zone.get(),
                          sysobj.get(),
@@ -367,12 +367,17 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver,
 
   meta.user.reset(RGWUserMetaHandlerAllocator::alloc(svc.user));
 
+  bucket.reset(new RGWBucketCtl(svc.zone,
+                                svc.bucket,
+                                svc.bucket_sync,
+                                svc.bi, svc.user));
+
   auto sync_module = svc.sync_modules->get_sync_module();
   if (sync_module) {
-    meta.bucket.reset(sync_module->alloc_bucket_meta_handler(rados));
+    meta.bucket = sync_module->alloc_bucket_meta_handler(rados, svc.bucket, bucket.get());
     meta.bucket_instance.reset(sync_module->alloc_bucket_instance_meta_handler(driver));
   } else {
-    meta.bucket.reset(RGWBucketMetaHandlerAllocator::alloc(rados));
+    meta.bucket = create_bucket_metadata_handler(rados, svc.bucket, bucket.get());
     meta.bucket_instance.reset(RGWBucketInstanceMetaHandlerAllocator::alloc(driver));
   }
 
@@ -385,15 +390,9 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver,
       *svc.sysobj, rados, svc.zone->get_zone_params());
 
   user.reset(new RGWUserCtl(svc.zone, svc.user, (RGWUserMetadataHandler *)meta.user.get()));
-  bucket.reset(new RGWBucketCtl(svc.zone,
-                                svc.bucket,
-                                svc.bucket_sync,
-                                svc.bi, svc.user));
 
-  RGWBucketMetadataHandlerBase *bucket_meta_handler = static_cast<RGWBucketMetadataHandlerBase *>(meta.bucket.get());
   RGWBucketInstanceMetadataHandlerBase *bi_meta_handler = static_cast<RGWBucketInstanceMetadataHandlerBase *>(meta.bucket_instance.get());
 
-  bucket_meta_handler->init(svc.bucket, bucket.get());
   bi_meta_handler->init(svc.zone, svc.bucket, svc.bi);
 
   meta.topic_cache = std::make_unique<RGWChainedCacheImpl<rgwrados::topic::cache_entry>>();
@@ -405,7 +404,6 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver,
 
   user->init(bucket.get());
   bucket->init(user.get(),
-               (RGWBucketMetadataHandler *)bucket_meta_handler,
                (RGWBucketInstanceMetadataHandler *)bi_meta_handler,
               svc.datalog_rados,
                dpp);
index a19248f2e9af53957cc976cc0bb49dc303d6ddf6..f99565ff55c5041afa04c3595b352f79499d7f15 100644 (file)
 
 #define dout_subsys ceph_subsys_rgw
 
-RGWMetadataHandler *RGWSyncModuleInstance::alloc_bucket_meta_handler(librados::Rados& rados)
+auto RGWSyncModuleInstance::alloc_bucket_meta_handler(librados::Rados& rados,
+                                                      RGWSI_Bucket* svc_bucket,
+                                                      RGWBucketCtl* ctl_bucket)
+    -> std::unique_ptr<RGWMetadataHandler>
 {
-  return RGWBucketMetaHandlerAllocator::alloc(rados);
+  return create_bucket_metadata_handler(rados, svc_bucket, ctl_bucket);
 }
 
 RGWBucketInstanceMetadataHandlerBase* RGWSyncModuleInstance::alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver)
index 4ca691ca665d53871b956872eaecdeab09e76a19..8f7b1d1f0c491e8b1bcb5c09d53a2da6ed462999 100644 (file)
@@ -44,6 +44,8 @@ public:
 class RGWRESTMgr;
 class RGWMetadataHandler;
 class RGWBucketInstanceMetadataHandlerBase;
+class RGWSI_Bucket;
+class RGWBucketCtl;
 
 class RGWSyncModuleInstance {
 public:
@@ -56,7 +58,10 @@ public:
   virtual bool supports_user_writes() {
     return false;
   }
-  virtual RGWMetadataHandler *alloc_bucket_meta_handler(librados::Rados& rados);
+  virtual auto alloc_bucket_meta_handler(librados::Rados& rados,
+                                         RGWSI_Bucket* svc_bucket,
+                                         RGWBucketCtl* ctl_bucket)
+      -> std::unique_ptr<RGWMetadataHandler>;
   virtual RGWBucketInstanceMetadataHandlerBase *alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver);
 
   // indication whether the sync module start with full sync (default behavior)
index 06ac2eb0866119323ded709eb59362505dfbfccd..540f57db8854f406f140ed3925740dcea2289cb1 100644 (file)
 
 #pragma once
 
+#include <memory>
 #include "rgw_service.h"
 
 #include "svc_bucket_types.h"
 
+class RGWMetadataLister;
+
 class RGWSI_Bucket : public RGWServiceInstance
 {
 public:
@@ -30,9 +33,12 @@ public:
   static std::string get_entrypoint_meta_key(const rgw_bucket& bucket);
   static std::string get_bi_meta_key(const rgw_bucket& bucket);
 
-  virtual RGWSI_Bucket_BE_Handler& get_ep_be_handler() = 0;
   virtual RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() = 0;
 
+  virtual int create_entrypoint_lister(const DoutPrefixProvider* dpp,
+                                       const std::string& marker,
+                                       std::unique_ptr<RGWMetadataLister>& lister) = 0;
+
   virtual int read_bucket_entrypoint_info(const std::string& key,
                                           RGWBucketEntryPoint *entry_point,
                                           RGWObjVersionTracker *objv_tracker,
index 0a9ddda8295e286119f5b498324537a4dd87a9b3..b4f0e28042196dd3c929e01710ebbe82f56e439f 100644 (file)
@@ -9,9 +9,11 @@
 #include "svc_bi.h"
 #include "svc_meta.h"
 #include "svc_meta_be_sobj.h"
+#include "svc_mdlog.h"
 #include "svc_sync_modules.h"
 
 #include "rgw_bucket.h"
+#include "rgw_metadata_lister.h"
 #include "rgw_string.h"
 #include "rgw_tools.h"
 #include "rgw_zone.h"
@@ -61,42 +63,6 @@ static std::string instance_oid_to_meta_key(const std::string& oid)
   return key;
 }
 
-class RGWSI_Bucket_SObj_Module : public RGWSI_MBSObj_Handler_Module {
-  RGWSI_Bucket_SObj::Svc& svc;
-
-  const string prefix;
-public:
-  RGWSI_Bucket_SObj_Module(RGWSI_Bucket_SObj::Svc& _svc) : RGWSI_MBSObj_Handler_Module("bucket"),
-                                                 svc(_svc) {}
-
-  void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override {
-    if (pool) {
-      *pool = svc.zone->get_zone_params().domain_root;
-    }
-    if (oid) {
-      *oid = key;
-    }
-  }
-
-  const string& get_oid_prefix() override {
-    return prefix;
-  }
-
-  bool is_valid_oid(const string& oid) override {
-    return (!oid.empty() && oid[0] != '.');
-  }
-
-  string key_to_oid(const string& key) override {
-    return key;
-  }
-
-  string oid_to_key(const string& oid) override {
-    /* should have been called after is_valid_oid(),
-     * so no need to check for validity */
-    return oid;
-  }
-};
-
 class RGWSI_BucketInstance_SObj_Module : public RGWSI_MBSObj_Handler_Module {
   RGWSI_Bucket_SObj::Svc& svc;
 
@@ -155,7 +121,8 @@ RGWSI_Bucket_SObj::~RGWSI_Bucket_SObj() {
 
 void RGWSI_Bucket_SObj::init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc,
                              RGWSI_SysObj_Cache *_cache_svc, RGWSI_BucketIndex *_bi,
-                             RGWSI_Meta *_meta_svc, RGWSI_MetaBackend *_meta_be_svc,
+                             RGWSI_Meta *_meta_svc, RGWSI_MDLog* mdlog_svc,
+                             RGWSI_MetaBackend *_meta_be_svc,
                              RGWSI_SyncModules *_sync_modules_svc,
                              RGWSI_Bucket_Sync *_bucket_sync_svc)
 {
@@ -165,6 +132,7 @@ void RGWSI_Bucket_SObj::init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc,
   svc.cache = _cache_svc;
   svc.bi = _bi;
   svc.meta = _meta_svc;
+  svc.mdlog = mdlog_svc;
   svc.meta_be = _meta_be_svc;
   svc.sync_modules = _sync_modules_svc;
   svc.bucket_sync = _bucket_sync_svc;
@@ -175,29 +143,11 @@ int RGWSI_Bucket_SObj::do_start(optional_yield, const DoutPrefixProvider *dpp)
   binfo_cache.reset(new RGWChainedCacheImpl<bucket_info_cache_entry>);
   binfo_cache->init(svc.cache);
 
-  /* create first backend handler for bucket entrypoints */
-
-  RGWSI_MetaBackend_Handler *ep_handler;
-
-  int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &ep_handler);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl;
-    return r;
-  }
-
-  ep_be_handler = ep_handler;
-
-  RGWSI_MetaBackend_Handler_SObj *ep_bh = static_cast<RGWSI_MetaBackend_Handler_SObj *>(ep_handler);
-
-  auto ep_module = new RGWSI_Bucket_SObj_Module(svc);
-  ep_be_module.reset(ep_module);
-  ep_bh->set_module(ep_module);
-
-  /* create a second backend handler for bucket instance */
+  /* create a backend handler for bucket instance */
 
   RGWSI_MetaBackend_Handler *bi_handler;
 
-  r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &bi_handler);
+  int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &bi_handler);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl;
     return r;
@@ -214,6 +164,41 @@ int RGWSI_Bucket_SObj::do_start(optional_yield, const DoutPrefixProvider *dpp)
   return 0;
 }
 
+
+class BucketEntrypointLister : public RGWMetadataLister {
+ public:
+  using RGWMetadataLister::RGWMetadataLister;
+
+  void filter_transform(std::vector<std::string>& oids,
+                        std::list<std::string>& keys) override
+  {
+    // bucket entrypoints and instances share a namespace, so filter out the
+    // instances based on prefix
+    constexpr auto filter = [] (const std::string& oid) {
+                              return oid.starts_with('.');
+                            };
+    // 'oids' is mutable so we can move its elements instead of copying
+    std::remove_copy_if(std::make_move_iterator(oids.begin()),
+                        std::make_move_iterator(oids.end()),
+                        std::back_inserter(keys), filter);
+  }
+};
+
+int RGWSI_Bucket_SObj::create_entrypoint_lister(
+    const DoutPrefixProvider* dpp,
+    const std::string& marker,
+    std::unique_ptr<RGWMetadataLister>& lister)
+{
+  const rgw_pool& pool = svc.zone->get_zone_params().domain_root;
+  auto p = std::make_unique<BucketEntrypointLister>(svc.sysobj->get_pool(pool));
+  int r = p->init(dpp, marker, ""); // empty prefix
+  if (r < 0) {
+    return r;
+  }
+  lister = std::move(p);
+  return 0;
+}
+
 int RGWSI_Bucket_SObj::read_bucket_entrypoint_info(const string& key,
                                                    RGWBucketEntryPoint *entry_point,
                                                    RGWObjVersionTracker *objv_tracker,
@@ -256,8 +241,13 @@ int RGWSI_Bucket_SObj::store_bucket_entrypoint_info(const string& key,
   encode(info, bl);
 
   const rgw_pool& pool = svc.zone->get_zone_params().domain_root;
-  return rgw_put_system_obj(dpp, svc.sysobj, pool, key, bl, exclusive,
-                            objv_tracker, mtime, y, pattrs);
+  int ret = rgw_put_system_obj(dpp, svc.sysobj, pool, key, bl, exclusive,
+                               objv_tracker, mtime, y, pattrs);
+  if (ret < 0) {
+    return ret;
+  }
+
+  return svc.mdlog->complete_entry(dpp, y, "bucket", key, objv_tracker);
 }
 
 int RGWSI_Bucket_SObj::remove_bucket_entrypoint_info(const string& key,
@@ -266,7 +256,12 @@ int RGWSI_Bucket_SObj::remove_bucket_entrypoint_info(const string& key,
                                                      const DoutPrefixProvider *dpp)
 {
   const rgw_pool& pool = svc.zone->get_zone_params().domain_root;
-  return rgw_delete_system_obj(dpp, svc.sysobj, pool, key, objv_tracker, y);
+  int ret = rgw_delete_system_obj(dpp, svc.sysobj, pool, key, objv_tracker, y);
+  if (ret < 0) {
+    return ret;
+  }
+
+  return svc.mdlog->complete_entry(dpp, y, "bucket", key, objv_tracker);
 }
 
 int RGWSI_Bucket_SObj::read_bucket_instance_info(const string& key,
index 60f109197438c9e2afe819ceb2fc81da4d0aa7de..3bf3a3a59ff1c4c507cefcb15f7524b373230d66 100644 (file)
@@ -28,6 +28,7 @@ class RGWSI_Zone;
 class RGWSI_SysObj;
 class RGWSI_SysObj_Cache;
 class RGWSI_Meta;
+class RGWSI_MDLog;
 class RGWSI_SyncModules;
 
 struct rgw_cache_entry_info;
@@ -46,8 +47,6 @@ class RGWSI_Bucket_SObj : public RGWSI_Bucket
   using RGWChainedCacheImpl_bucket_info_cache_entry = RGWChainedCacheImpl<bucket_info_cache_entry>;
   std::unique_ptr<RGWChainedCacheImpl_bucket_info_cache_entry> binfo_cache;
 
-  RGWSI_Bucket_BE_Handler ep_be_handler;
-  std::unique_ptr<RGWSI_MetaBackend::Module> ep_be_module;
   RGWSI_BucketInstance_BE_Handler bi_be_handler;
   std::unique_ptr<RGWSI_MetaBackend::Module> bi_be_module;
 
@@ -75,6 +74,7 @@ public:
     RGWSI_SysObj *sysobj{nullptr};
     RGWSI_SysObj_Cache *cache{nullptr};
     RGWSI_Meta *meta{nullptr};
+    RGWSI_MDLog *mdlog{nullptr};
     RGWSI_MetaBackend *meta_be{nullptr};
     RGWSI_SyncModules *sync_modules{nullptr};
     RGWSI_Bucket_Sync *bucket_sync{nullptr};
@@ -83,10 +83,6 @@ public:
   RGWSI_Bucket_SObj(CephContext *cct);
   ~RGWSI_Bucket_SObj();
 
-  RGWSI_Bucket_BE_Handler& get_ep_be_handler() override {
-    return ep_be_handler;
-  }
-
   RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() override {
     return bi_be_handler;
   }
@@ -96,10 +92,14 @@ public:
            RGWSI_SysObj_Cache *_cache_svc,
             RGWSI_BucketIndex *_bi,
             RGWSI_Meta *_meta_svc,
+            RGWSI_MDLog *mdlog_svc,
             RGWSI_MetaBackend *_meta_be_svc,
            RGWSI_SyncModules *_sync_modules_svc,
            RGWSI_Bucket_Sync *_bucket_sync_svc);
 
+  int create_entrypoint_lister(const DoutPrefixProvider* dpp,
+                               const std::string& marker,
+                               std::unique_ptr<RGWMetadataLister>& lister) override;
 
   int read_bucket_entrypoint_info(const std::string& key,
                                   RGWBucketEntryPoint *entry_point,
index 03a967d12673bd9dd9969cf671f8c23a5b2d9871..90b9b6611d270ed555c50480428591f4233026fb 100644 (file)
@@ -1,6 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#include <fmt/format.h>
+
 #include "svc_mdlog.h"
 #include "svc_zone.h"
 #include "svc_sys_obj.h"