From ae1c2689cda6a19b4540fc3e2530f5b364cd7d4f Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Wed, 2 Nov 2022 20:06:23 +0000 Subject: [PATCH] rgw: add 'inline_data' zone placement info option Adds a new RGW zone placement info option to control whether an object's first data chunk is placed in the head object. This allows admins to make a tradeoff between optimizing for PUT/GET performance vs. DELETE performance for some cluster configurations. Fixes: https://tracker.ceph.com/issues/57965 Signed-off-by: Cory Snyder (cherry picked from commit 9052ca420f003af4b89b568c50bd5083333d3010) Conflicts: src/rgw/rgw_zone.cc src/rgw/rgw_zone_types.h Cherry-pick notes: - Conflicts due to rgw_zone_types.h moving to rgw_zone.h --- doc/man/8/radosgw-admin.rst | 4 ++++ doc/radosgw/placement.rst | 4 ++-- src/rgw/rgw_admin.cc | 12 ++++++++++++ src/rgw/rgw_putobj_processor.cc | 8 +++++++- src/rgw/rgw_zone.cc | 2 ++ src/rgw/rgw_zone.h | 11 ++++++++--- src/test/cli/radosgw-admin/help.t | 3 +++ 7 files changed, 38 insertions(+), 6 deletions(-) diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst index 57d7fcfc90238..e099bba074df0 100644 --- a/doc/man/8/radosgw-admin.rst +++ b/doc/man/8/radosgw-admin.rst @@ -743,6 +743,10 @@ Options The placement target index type (normal, indexless, or #id). +.. option:: --placement-inline-data= + + Whether the placement target is configured to store a data chunk inline in head objects. + .. option:: --tier-type= The zone tier type. diff --git a/doc/radosgw/placement.rst b/doc/radosgw/placement.rst index d799244da4e2f..ae4316524d94a 100644 --- a/doc/radosgw/placement.rst +++ b/doc/radosgw/placement.rst @@ -86,7 +86,8 @@ The zone placement configuration can be queried with: } }, "data_extra_pool": "default.rgw.buckets.non-ec", - "index_type": 0 + "index_type": 0, + "inline_data": true } } ], @@ -132,7 +133,6 @@ Then provide the zone placement info for that target: the BlueStore DB is located on faster storage than bucket data since it eliminates the need to access slower devices synchronously while processing the client request. In that case, data associated with the deleted objects is removed asynchronously in the background by garbage collection. - .. _adding_a_storage_class: Adding a Storage Class diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 097ad1c2f90b3..fd8188e2ddabb 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -369,6 +369,9 @@ void usage() cout << " --data-extra-pool= placement target data extra (non-ec) pool\n"; cout << " --placement-index-type=\n"; cout << " placement target index type (normal, indexless, or #id)\n"; + cout << " --placement-inline-data=\n"; + cout << " set whether the placement target is configured to store a data\n"; + cout << " chunk inline in head objects\n"; cout << " --compression= placement target compression type (plugin name or empty/none)\n"; cout << " --tier-type= zone tier type\n"; cout << " --tier-config==[,...]\n"; @@ -3478,6 +3481,8 @@ int main(int argc, const char **argv) list tags; list tags_add; list tags_rm; + int placement_inline_data = true; + bool placement_inline_data_specified = false; int64_t max_objects = -1; int64_t max_size = -1; @@ -3515,6 +3520,7 @@ int main(int argc, const char **argv) int num_shards = 0; bool num_shards_specified = false; std::optional bucket_index_max_shards; + int max_concurrent_ios = 32; uint64_t orphan_stale_secs = (24 * 3600); int detail = false; @@ -3843,6 +3849,9 @@ int main(int argc, const char **argv) // do nothing } else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) { // do nothing + } else if (ceph_argparse_binary_flag(args, i, &placement_inline_data, NULL, "--placement-inline-data", (char*)NULL)) { + placement_inline_data_specified = true; + // do nothing } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) { caps = val; } else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) { @@ -5977,6 +5986,9 @@ int main(int argc, const char **argv) if (index_type_specified) { info.index_type = placement_index_type; } + if (placement_inline_data_specified) { + info.inline_data = placement_inline_data; + } ret = check_pool_support_omap(info.get_data_extra_pool()); if (ret < 0) { diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 5c21c76a65b8d..f0045d6f1686a 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -18,6 +18,7 @@ #include "rgw_multi.h" #include "rgw_compression.h" #include "services/svc_sys_obj.h" +#include "services/svc_zone.h" #include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw @@ -246,7 +247,12 @@ int AtomicObjectProcessor::prepare(optional_yield y) } if (same_pool) { - head_max_size = max_head_chunk_size; + RGWZonePlacementInfo placement_info; + if (!store->svc()->zone->get_zone_params().get_placement(head_obj->get_bucket()->get_placement_rule().name, &placement_info) || placement_info.inline_data) { + head_max_size = max_head_chunk_size; + } else { + head_max_size = 0; + } chunk_size = max_head_chunk_size; } diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index c8e8d09b8879a..bbeea7efb84e1 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -2607,6 +2607,7 @@ void RGWZonePlacementInfo::dump(Formatter *f) const encode_json("storage_classes", storage_classes, f); encode_json("data_extra_pool", data_extra_pool, f); encode_json("index_type", (uint32_t)index_type, f); + encode_json("inline_data", inline_data, f); /* no real need for backward compatibility of compression_type and data_pool in here, * rather not clutter the output */ @@ -2619,6 +2620,7 @@ void RGWZonePlacementInfo::decode_json(JSONObj *obj) JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj); uint32_t it; JSONDecoder::decode_json("index_type", it, obj); + JSONDecoder::decode_json("inline_data", inline_data, obj); index_type = (rgw::BucketIndexType)it; /* backward compatibility, these are now defined in storage_classes */ diff --git a/src/rgw/rgw_zone.h b/src/rgw/rgw_zone.h index 9e89cbf150a72..f8ddcec5bda51 100644 --- a/src/rgw/rgw_zone.h +++ b/src/rgw/rgw_zone.h @@ -275,11 +275,12 @@ struct RGWZonePlacementInfo { rgw_pool data_extra_pool; /* if not set we should use data_pool */ RGWZoneStorageClasses storage_classes; rgw::BucketIndexType index_type; + bool inline_data; - RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal) {} + RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal), inline_data(true) {} void encode(bufferlist& bl) const { - ENCODE_START(7, 1, bl); + ENCODE_START(8, 1, bl); encode(index_pool.to_str(), bl); rgw_pool standard_data_pool = get_data_pool(RGW_STORAGE_CLASS_STANDARD); encode(standard_data_pool.to_str(), bl); @@ -288,11 +289,12 @@ struct RGWZonePlacementInfo { std::string standard_compression_type = get_compression_type(RGW_STORAGE_CLASS_STANDARD); encode(standard_compression_type, bl); encode(storage_classes, bl); + encode(inline_data, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(7, bl); + DECODE_START(8, bl); std::string index_pool_str; std::string data_pool_str; decode(index_pool_str, bl); @@ -319,6 +321,9 @@ struct RGWZonePlacementInfo { storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &standard_data_pool, (!standard_compression_type.empty() ? &standard_compression_type : nullptr)); } + if (struct_v >= 8) { + decode(inline_data, bl); + } DECODE_FINISH(bl); } const rgw_pool& get_data_extra_pool() const { diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 6784b966ca445..f6ba9650fddd6 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -261,6 +261,9 @@ --data-extra-pool= placement target data extra (non-ec) pool --placement-index-type= placement target index type (normal, indexless, or #id) + --placement-inline-data= + set whether the placement target is configured to store a data + chunk inline in head objects --compression= placement target compression type (plugin name or empty/none) --tier-type= zone tier type --tier-config==[,...] -- 2.39.5