From ed717551142df75d29f9a84cd6d80b549efcbd25 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Wed, 20 Sep 2023 16:02:20 -0400 Subject: [PATCH] rgw/multisite: draft of bucket index cleanup after deletion tracker: https://tracker.ceph.com/issues/20802 Signed-off-by: Shilpa Jagannath --- src/rgw/driver/rados/rgw_rados.cc | 7 +++++++ src/rgw/driver/rados/rgw_trim_bilog.cc | 20 ++++++++++++++++---- src/rgw/rgw_bucket_layout.cc | 13 ++++++++++++- src/rgw/rgw_bucket_layout.h | 9 +++++++++ src/rgw/rgw_common.cc | 3 +++ src/rgw/rgw_common.h | 1 + src/rgw/rgw_metadata.cc | 2 ++ 7 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index c7bc3d3afde04..109dde79603be 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5612,6 +5612,13 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& ob (void) CLSRGWIssueBucketIndexClean(index_pool, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); + } else { + bucket_info.deleted = true; + map attrs; + r = put_bucket_instance_info(bucket_info, false, real_time(), &attrs, dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: put_bucket_info on bucket=" << bucket_info.bucket.name << " returned err=" << r << dendl; + } } return 0; diff --git a/src/rgw/driver/rados/rgw_trim_bilog.cc b/src/rgw/driver/rados/rgw_trim_bilog.cc index d9960289a8519..d2f86abe8b929 100644 --- a/src/rgw/driver/rados/rgw_trim_bilog.cc +++ b/src/rgw/driver/rados/rgw_trim_bilog.cc @@ -534,6 +534,7 @@ private: return 0; } + public: BucketTrimInstanceCR(rgw::sal::RadosStore* store, RGWHTTPManager *http, BucketTrimObserver *observer, @@ -758,14 +759,25 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp) << cpp_strerror(retcode) << dendl; return set_cr_error(retcode); } + + //remove bucket instance metadata + if (clean_info->first.layout.logs.front().layout.type == rgw::BucketLogType::Deleted) { + retcode = store->ctl()->bucket->remove_bucket_instance_info(bucket, clean_info->first, null_yield, dpp); + if (retcode < 0) { + ldpp_dout(dpp, 0) << "failed to remove instance bucket info: " + << cpp_strerror(retcode) << dendl; + return set_cr_error(retcode); + } + } + clean_info = std::nullopt; } } else { if (totrim.layout.type != rgw::BucketLogType::InIndex) { - ldpp_dout(dpp, 0) << "Unable to convert log of unknown type " - << totrim.layout.type - << " to rgw::bucket_index_layout_generation " << dendl; - return set_cr_error(-EINVAL); + ldpp_dout(dpp, 0) << "Unable to convert log of unknown type " + << totrim.layout.type + << " to rgw::bucket_index_layout_generation " << dendl; + return set_cr_error(-EINVAL); } // To avoid hammering the OSD too hard, either trim old // generations OR trim the current one. diff --git a/src/rgw/rgw_bucket_layout.cc b/src/rgw/rgw_bucket_layout.cc index 10ff7200eaa30..dd62f7e4e1803 100644 --- a/src/rgw/rgw_bucket_layout.cc +++ b/src/rgw/rgw_bucket_layout.cc @@ -187,6 +187,7 @@ std::string_view to_string(const BucketLogType& t) { switch (t) { case BucketLogType::InIndex: return "InIndex"; + case BucketLogType::Deleted: return "Deleted"; default: return "Unknown"; } } @@ -196,6 +197,10 @@ bool parse(std::string_view str, BucketLogType& t) t = BucketLogType::InIndex; return true; } + if (boost::iequals(str, "Deleted")) { + t = BucketLogType::Deleted; + return true; + } return false; } void encode_json_impl(const char *name, const BucketLogType& t, ceph::Formatter *f) @@ -246,6 +251,8 @@ void encode(const bucket_log_layout& l, bufferlist& bl, uint64_t f) case BucketLogType::InIndex: encode(l.in_index, bl); break; + case BucketLogType::Deleted: + break; } ENCODE_FINISH(bl); } @@ -257,6 +264,8 @@ void decode(bucket_log_layout& l, bufferlist::const_iterator& bl) case BucketLogType::InIndex: decode(l.in_index, bl); break; + case BucketLogType::Deleted: + break; } DECODE_FINISH(bl); } @@ -272,7 +281,9 @@ void encode_json_impl(const char *name, const bucket_log_layout& l, ceph::Format void decode_json_obj(bucket_log_layout& l, JSONObj *obj) { JSONDecoder::decode_json("type", l.type, obj); - JSONDecoder::decode_json("in_index", l.in_index, obj); + if (l.type == BucketLogType::InIndex) { + JSONDecoder::decode_json("in_index", l.in_index, obj); + } } // bucket_log_layout_generation diff --git a/src/rgw/rgw_bucket_layout.h b/src/rgw/rgw_bucket_layout.h index 08bacc81b306b..77cbef946028e 100644 --- a/src/rgw/rgw_bucket_layout.h +++ b/src/rgw/rgw_bucket_layout.h @@ -136,6 +136,7 @@ void decode_json_obj(bucket_index_layout_generation& l, JSONObj *obj); enum class BucketLogType : uint8_t { // colocated with bucket index, so the log layout matches the index layout InIndex, + Deleted }; std::string_view to_string(const BucketLogType& t); @@ -148,6 +149,8 @@ inline std::ostream& operator<<(std::ostream& out, const BucketLogType &log_type switch (log_type) { case BucketLogType::InIndex: return out << "InIndex"; + case BucketLogType::Deleted: + return out << "Deleted"; default: return out << "Unknown"; } @@ -213,6 +216,12 @@ inline auto matches_gen(uint64_t gen) return [gen] (const bucket_log_layout_generation& l) { return l.gen == gen; }; } +inline bucket_log_layout_generation log_layout_from_deleted_index( + uint64_t gen, const bucket_index_layout_generation& index) +{ + return {gen, {BucketLogType::Deleted, {index.gen, index.layout.normal}}}; +} + inline bucket_index_layout_generation log_to_index_layout(const bucket_log_layout_generation& log_layout) { ceph_assert(log_layout.layout.type == BucketLogType::InIndex); diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 97e8973b16257..5eda2b96f215c 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -2557,6 +2557,7 @@ void RGWBucketInfo::dump(Formatter *f) const encode_json("mdsearch_config", mdsearch_config, f); encode_json("reshard_status", (int)reshard_status, f); encode_json("new_bucket_instance_id", new_bucket_instance_id, f); + encode_json("deleted", deleted, f); if (!empty_sync_policy()) { encode_json("sync_policy", *sync_policy, f); } @@ -2597,6 +2598,8 @@ void RGWBucketInfo::decode_json(JSONObj *obj) { int rs; JSONDecoder::decode_json("reshard_status", rs, obj); reshard_status = (cls_rgw_reshard_status)rs; + JSONDecoder::decode_json("deleted", deleted, obj); + rgw_sync_policy_info sp; JSONDecoder::decode_json("sync_policy", sp, obj); if (!sp.empty()) { diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index a7d3bbef62480..a4a3a412791c4 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1075,6 +1075,7 @@ struct RGWBucketInfo { bool has_instance_obj{false}; RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */ RGWQuotaInfo quota; + bool deleted{false}; // layout of bucket index objects rgw::BucketLayout layout; diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index d84ff16f71117..2323bd1d1579a 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -353,12 +353,14 @@ int RGWMetadataManager::remove(string& metadata_key, optional_yield y, const Dou int ret = find_handler(metadata_key, &handler, entry); if (ret < 0) { + ldout(cct, 10) << "ERROR: " << __func__ << "(): find_handler returned: ret=" << ret << dendl; return ret; } RGWMetadataObject *obj; ret = handler->get(entry, &obj, y, dpp); if (ret < 0) { + ldout(cct, 10) << "ERROR: " << __func__ << "(): handler->get() returned: ret=" << ret << dendl; return ret; } RGWObjVersionTracker objv_tracker; -- 2.39.5