From 14727dd19e346c2e4eb7b031d4fb37435dc7f263 Mon Sep 17 00:00:00 2001 From: liangmingyuan Date: Thu, 8 Feb 2024 15:48:15 +0800 Subject: [PATCH] rgw/reshard: Define the operation to record a duplicated index entry. Signed-off-by: Mingyuan Liang --- src/cls/rgw/cls_rgw.cc | 38 ++++++++++++++++++++- src/cls/rgw/cls_rgw_types.cc | 51 ++++++++++++++++++++++++++++- src/cls/rgw/cls_rgw_types.h | 28 +++++++++++++--- src/rgw/rgw_admin.cc | 2 ++ src/tools/ceph-dencoder/rgw_types.h | 1 + 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 27a484dd51e..cec5cbe243d 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -54,13 +54,15 @@ CLS_NAME(rgw) #define BI_BUCKET_LOG_INDEX 1 #define BI_BUCKET_OBJ_INSTANCE_INDEX 2 #define BI_BUCKET_OLH_DATA_INDEX 3 +#define BI_BUCKET_RESHARD_LOG_INDEX 4 -#define BI_BUCKET_LAST_INDEX 4 +#define BI_BUCKET_LAST_INDEX 5 static std::string bucket_index_prefixes[] = { "", /* special handling for the objs list index */ "0_", /* bucket log index */ "1000_", /* obj instance index */ "1001_", /* olh data index */ + "2001_", /* reshard log index */ /* this must be the last index */ "9999_",}; @@ -133,6 +135,40 @@ static void get_index_ver_key(cls_method_context_t hctx, uint64_t index_ver, str *key = buf; } +static void bi_reshard_log_prefix(string& key) +{ + key = BI_PREFIX_CHAR; + key.append(bucket_index_prefixes[BI_BUCKET_RESHARD_LOG_INDEX]); +} + +// 0x802001_idx +static void bi_reshard_log_key(cls_method_context_t hctx, string& key, string& idx) +{ + bi_reshard_log_prefix(key); + key.append(idx); +} + +static int reshard_log_index_operation(cls_method_context_t hctx, string& idx, + cls_rgw_obj_key& key, bufferlist* log_bl) +{ + string reshard_log_idx; + bi_reshard_log_key(hctx, reshard_log_idx, idx); + + rgw_cls_bi_entry reshard_log_entry; + if (log_bl && log_bl->length() == 0) { + reshard_log_entry.type = BIIndexType::ReshardDeleted; + rgw_bucket_deleted_entry delete_entry; + delete_entry.key = key; + encode(delete_entry, reshard_log_entry.data); + } else { + reshard_log_entry.data = *log_bl; + } + reshard_log_entry.idx = idx; + bufferlist bl; + encode(reshard_log_entry, bl); + return cls_cxx_map_set_val(hctx, reshard_log_idx, &bl); +} + static void bi_log_prefix(string& key) { key = BI_PREFIX_CHAR; diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc index ccf92c3765f..b58d3769106 100644 --- a/src/cls/rgw/cls_rgw_types.cc +++ b/src/cls/rgw/cls_rgw_types.cc @@ -312,6 +312,13 @@ static void dump_bi_entry(bufferlist bl, BIIndexType index_type, Formatter *form encode_json("entry", entry, formatter); } break; + case BIIndexType::ReshardDeleted: + { + rgw_bucket_deleted_entry entry; + decode(entry, iter); + encode_json("entry", entry, formatter); + } + break; default: break; } @@ -327,6 +334,8 @@ void rgw_cls_bi_entry::decode_json(JSONObj *obj, cls_rgw_obj_key *effective_key) type = BIIndexType::Instance; } else if (s == "olh") { type = BIIndexType::OLH; + } else if (s == "resharddeleted") { + type = BIIndexType::ReshardDeleted; } else { type = BIIndexType::Invalid; } @@ -355,6 +364,17 @@ void rgw_cls_bi_entry::decode_json(JSONObj *obj, cls_rgw_obj_key *effective_key) } } break; + case BIIndexType::ReshardDeleted: + { + rgw_bucket_deleted_entry entry; + JSONDecoder::decode_json("entry", entry, obj); + encode(entry, data); + + if (effective_key) { + *effective_key = entry.key; + } + } + break; default: break; } @@ -373,6 +393,9 @@ void rgw_cls_bi_entry::dump(Formatter *f) const case BIIndexType::OLH: type_str = "olh"; break; + case BIIndexType::ReshardDeleted: + type_str = "resharddeleted"; + break; default: type_str = "invalid"; } @@ -390,7 +413,13 @@ bool rgw_cls_bi_entry::get_info(cls_rgw_obj_key *key, if (type == BIIndexType::OLH) { rgw_bucket_olh_entry entry; decode(entry, iter); - *key = entry.key; + *key = std::move(entry.key); + return false; + } + if (type == BIIndexType::ReshardDeleted) { + rgw_bucket_deleted_entry entry; + decode(entry, iter); + *key = std::move(entry.key); return false; } @@ -465,6 +494,25 @@ void rgw_bucket_olh_entry::generate_test_instances(list& o.push_back(new rgw_bucket_olh_entry); } +void rgw_bucket_deleted_entry::dump(Formatter *f) const +{ + encode_json("key", key, f); +} + +void rgw_bucket_deleted_entry::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("key", key, obj); +} + +void rgw_bucket_deleted_entry::generate_test_instances(list& o) +{ + rgw_bucket_deleted_entry *entry = new rgw_bucket_deleted_entry; + entry->key.name = "key.name"; + entry->key.instance = "key.instance"; + o.push_back(entry); + o.push_back(new rgw_bucket_deleted_entry); +} + void rgw_bucket_olh_log_entry::generate_test_instances(list& o) { rgw_bucket_olh_log_entry *entry = new rgw_bucket_olh_log_entry; @@ -519,6 +567,7 @@ void rgw_bucket_olh_log_entry::decode_json(JSONObj *obj) JSONDecoder::decode_json("key", key, obj); JSONDecoder::decode_json("delete_marker", delete_marker, obj); } + void rgw_bi_log_entry::decode_json(JSONObj *obj) { JSONDecoder::decode_json("op_id", id, obj); diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index de10671e290..9cc1ff1a580 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -469,10 +469,11 @@ struct rgw_bucket_dir_entry { WRITE_CLASS_ENCODER(rgw_bucket_dir_entry) enum class BIIndexType : uint8_t { - Invalid = 0, - Plain = 1, - Instance = 2, - OLH = 3, + Invalid = 0, + Plain = 1, + Instance = 2, + OLH = 3, + ReshardDeleted = 4, }; struct rgw_bucket_category_stats; @@ -592,6 +593,25 @@ struct rgw_bucket_olh_entry { }; WRITE_CLASS_ENCODER(rgw_bucket_olh_entry) +struct rgw_bucket_deleted_entry { + cls_rgw_obj_key key; + rgw_bucket_deleted_entry() {} + void encode(ceph::buffer::list &bl) const { + ENCODE_START(1, 1, bl); + encode(key, bl); + ENCODE_FINISH(bl); + } + void decode(ceph::buffer::list::const_iterator &bl) { + DECODE_START(1, bl); + decode(key, bl); + DECODE_FINISH(bl); + } + void dump(ceph::Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); +}; +WRITE_CLASS_ENCODER(rgw_bucket_deleted_entry) + struct rgw_bi_log_entry { std::string id; std::string object; diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 0008339526b..44900e5d1af 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1150,6 +1150,8 @@ BIIndexType get_bi_index_type(const string& type_str) { return BIIndexType::Instance; if (type_str == "olh") return BIIndexType::OLH; + if (type_str == "resharddeleted") + return BIIndexType::ReshardDeleted; return BIIndexType::Invalid; } diff --git a/src/tools/ceph-dencoder/rgw_types.h b/src/tools/ceph-dencoder/rgw_types.h index 8cd0acfc624..af5ebb2f280 100644 --- a/src/tools/ceph-dencoder/rgw_types.h +++ b/src/tools/ceph-dencoder/rgw_types.h @@ -52,6 +52,7 @@ TYPE(rgw_bucket_olh_log_entry) TYPE(rgw_usage_log_entry) TYPE(rgw_cls_bi_entry) TYPE(rgw_bucket_olh_entry) +TYPE(rgw_bucket_deleted_entry) TYPE(rgw_usage_data) TYPE(rgw_s3select_usage_data) TYPE(rgw_usage_log_info) -- 2.39.5