From: Casey Bodley Date: Tue, 3 Mar 2020 20:42:54 +0000 (-0500) Subject: rgw: add BucketLayout to RGWBucketInfo X-Git-Tag: v16.1.0~2727^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=60af77262a320646eda60608f9caacec3cf53d85;p=ceph.git rgw: add BucketLayout to RGWBucketInfo Signed-off-by: Casey Bodley --- diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index bfacafc41e86..7cbd18da3bba 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -58,6 +58,7 @@ set(librgw_common_srcs rgw_arn.cc rgw_basic_types.cc rgw_bucket.cc + rgw_bucket_layout.cc rgw_bucket_sync.cc rgw_cache.cc rgw_common.cc diff --git a/src/rgw/rgw_bucket_layout.cc b/src/rgw/rgw_bucket_layout.cc new file mode 100644 index 000000000000..942d113c35be --- /dev/null +++ b/src/rgw/rgw_bucket_layout.cc @@ -0,0 +1,94 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "rgw_bucket_layout.h" + +namespace rgw { + +void encode(const bucket_index_normal_layout& l, bufferlist& bl, uint64_t f) +{ + ENCODE_START(1, 1, bl); + encode(l.num_shards, bl); + encode(l.hash_type, bl); + ENCODE_FINISH(bl); +} +void decode(bucket_index_normal_layout& l, bufferlist::const_iterator& bl) +{ + DECODE_START(1, bl); + decode(l.num_shards, bl); + decode(l.hash_type, bl); + DECODE_FINISH(bl); +} + +void encode(const bucket_index_layout& l, bufferlist& bl, uint64_t f) +{ + ENCODE_START(1, 1, bl); + encode(l.type, bl); + switch (l.type) { + case BucketIndexType::Normal: + encode(l.normal, bl); + break; + case BucketIndexType::Indexless: + break; + } + ENCODE_FINISH(bl); +} +void decode(bucket_index_layout& l, bufferlist::const_iterator& bl) +{ + DECODE_START(1, bl); + decode(l.type, bl); + switch (l.type) { + case BucketIndexType::Normal: + decode(l.normal, bl); + break; + case BucketIndexType::Indexless: + break; + } + DECODE_FINISH(bl); +} + +void encode(const bucket_index_layout_generation& l, bufferlist& bl, uint64_t f) +{ + ENCODE_START(1, 1, bl); + encode(l.gen, bl); + encode(l.layout, bl); + ENCODE_FINISH(bl); +} +void decode(bucket_index_layout_generation& l, bufferlist::const_iterator& bl) +{ + DECODE_START(1, bl); + decode(l.gen, bl); + decode(l.layout, bl); + DECODE_FINISH(bl); +} + +void encode(const BucketLayout& l, bufferlist& bl, uint64_t f) +{ + ENCODE_START(1, 1, bl); + encode(l.resharding, bl); + encode(l.current_index, bl); + encode(l.target_index, bl); + ENCODE_FINISH(bl); +} +void decode(BucketLayout& l, bufferlist::const_iterator& bl) +{ + DECODE_START(1, bl); + decode(l.resharding, bl); + decode(l.current_index, bl); + decode(l.target_index, bl); + DECODE_FINISH(bl); +} + +} // namespace rgw diff --git a/src/rgw/rgw_bucket_layout.h b/src/rgw/rgw_bucket_layout.h new file mode 100644 index 000000000000..5f9d156c98cf --- /dev/null +++ b/src/rgw/rgw_bucket_layout.h @@ -0,0 +1,82 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include +#include +#include "include/encoding.h" + +namespace rgw { + +enum class BucketIndexType : uint8_t { + Normal, // normal hash-based sharded index layout + Indexless, // no bucket index, so listing is unsupported +}; + +enum class BucketHashType : uint8_t { + Mod, // rjenkins hash of object name, modulo num_shards +}; + +struct bucket_index_normal_layout { + uint32_t num_shards = 1; + + BucketHashType hash_type = BucketHashType::Mod; +}; + +void encode(const bucket_index_normal_layout& l, bufferlist& bl, uint64_t f=0); +void decode(bucket_index_normal_layout& l, bufferlist::const_iterator& bl); + + +struct bucket_index_layout { + BucketIndexType type = BucketIndexType::Normal; + + // TODO: variant of layout types? + bucket_index_normal_layout normal; +}; + +void encode(const bucket_index_layout& l, bufferlist& bl, uint64_t f=0); +void decode(bucket_index_layout& l, bufferlist::const_iterator& bl); + + +struct bucket_index_layout_generation { + uint64_t gen = 0; + bucket_index_layout layout; +}; + +void encode(const bucket_index_layout_generation& l, bufferlist& bl, uint64_t f=0); +void decode(bucket_index_layout_generation& l, bufferlist::const_iterator& bl); + + +enum class BucketReshardState : uint8_t { + None, + InProgress, +}; + +// describes the layout of bucket index objects +struct BucketLayout { + BucketReshardState resharding = BucketReshardState::None; + + // current bucket index layout + bucket_index_layout_generation current_index; + + // target index layout of a resharding operation + std::optional target_index; +}; + +void encode(const BucketLayout& l, bufferlist& bl, uint64_t f=0); +void decode(BucketLayout& l, bufferlist::const_iterator& bl); + +} // namespace rgw diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index f440bb226122..f3c08f8873c3 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -1990,7 +1990,7 @@ RGWBucketInfo::~RGWBucketInfo() } void RGWBucketInfo::encode(bufferlist& bl) const { - ENCODE_START(21, 4, bl); + ENCODE_START(22, 4, bl); encode(bucket, bl); encode(owner.id, bl); encode(flags, bl); @@ -2025,11 +2025,12 @@ void RGWBucketInfo::encode(bufferlist& bl) const { if (has_sync_policy) { encode(*sync_policy, bl); } + encode(layout, bl); ENCODE_FINISH(bl); } void RGWBucketInfo::decode(bufferlist::const_iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN_32(21, 4, 4, bl); + DECODE_START_LEGACY_COMPAT_LEN_32(22, 4, 4, bl); decode(bucket, bl); if (struct_v >= 2) { string s; @@ -2099,6 +2100,9 @@ void RGWBucketInfo::decode(bufferlist::const_iterator& bl) { if (struct_v >= 21) { decode(sync_policy, bl); } + if (struct_v >= 22) { + decode(layout, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 8d840731fe07..34fb71b833b9 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -24,6 +24,7 @@ #include "common/ceph_crypto.h" #include "common/random_string.h" #include "rgw_acl.h" +#include "rgw_bucket_layout.h" #include "rgw_cors.h" #include "rgw_iam_policy.h" #include "rgw_quota.h" @@ -1124,6 +1125,9 @@ struct RGWBucketInfo { RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */ RGWQuotaInfo quota; + // layout of bucket index objects + rgw::BucketLayout layout; + // Represents the number of bucket index object shards: // - value of 0 indicates there is no sharding (this is by default // before this feature is implemented).