From 9052ca420f003af4b89b568c50bd5083333d3010 Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Wed, 2 Nov 2022 20:06:23 +0000 Subject: [PATCH] rgw: add 'inline_data' zone placement info option Adds a new RGW zone placement info option to control whether an object's first data chunk is placed in the head object. This allows admins to make a tradeoff between optimizing for PUT/GET performance vs. DELETE performance for some cluster configurations. Fixes: https://tracker.ceph.com/issues/57965 Signed-off-by: Cory Snyder --- doc/man/8/radosgw-admin.rst | 4 ++++ doc/radosgw/placement.rst | 13 ++++++++++++- src/rgw/rgw_admin.cc | 12 ++++++++++++ src/rgw/rgw_putobj_processor.cc | 7 ++++++- src/rgw/rgw_zone.cc | 3 +++ src/rgw/rgw_zone_types.h | 11 ++++++++--- src/test/cli/radosgw-admin/help.t | 3 +++ 7 files changed, 48 insertions(+), 5 deletions(-) diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst index a6609c56dce3c..54a66e17a0b8a 100644 --- a/doc/man/8/radosgw-admin.rst +++ b/doc/man/8/radosgw-admin.rst @@ -743,6 +743,10 @@ Options The placement target index type (normal, indexless, or #id). +.. option:: --placement-inline-data= + + Whether the placement target is configured to store a data chunk inline in head objects. + .. option:: --tier-type= The zone tier type. diff --git a/doc/radosgw/placement.rst b/doc/radosgw/placement.rst index 595fde2947040..6274b022f4a00 100644 --- a/doc/radosgw/placement.rst +++ b/doc/radosgw/placement.rst @@ -86,7 +86,8 @@ The zone placement configuration can be queried with: } }, "data_extra_pool": "default.rgw.buckets.non-ec", - "index_type": 0 + "index_type": 0, + "inline_data": true } } ], @@ -123,6 +124,16 @@ Then provide the zone placement info for that target: --index-pool default.rgw.temporary.index \ --data-extra-pool default.rgw.temporary.non-ec +.. note:: With default placement target settings, RGW stores an object's first data chunk in the RADOS "head" object along + with xattr metadata. The `--placement-inline-data=false` flag may be passed with the `zone placement add` or + `zone placement modify` commands to change this behavior for new objects stored on the target. + When data is stored inline (default), it may provide an advantage for read/write workloads since the first chunk of + an object's data can be retrieved/stored in a single librados call along with object metadata. On the other hand, a + target that does not store data inline can provide a performance benefit for RGW client delete requests when + bluestore db is located on faster storage devices (as compared to data devices) since it eliminates the need to access + slower devices synchronously while processing the client request. In that case, all data associated with the deleted + objects can be removed asynchronously in the background by garbage collection. + .. _adding_a_storage_class: Adding a Storage Class diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index f18ce8cfbfb78..46b6ced3cdeda 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -392,6 +392,9 @@ void usage() cout << " --data-extra-pool= placement target data extra (non-ec) pool\n"; cout << " --placement-index-type=\n"; cout << " placement target index type (normal, indexless, or #id)\n"; + cout << " --placement-inline-data=\n"; + cout << " set whether the placement target is configured to store a data\n"; + cout << " chunk inline in head objects\n"; cout << " --compression= placement target compression type (plugin name or empty/none)\n"; cout << " --tier-type= zone tier type\n"; cout << " --tier-config==[,...]\n"; @@ -3537,6 +3540,8 @@ int main(int argc, const char **argv) list tags; list tags_add; list tags_rm; + int placement_inline_data = true; + bool placement_inline_data_specified = false; int64_t max_objects = -1; int64_t max_size = -1; @@ -3574,6 +3579,7 @@ int main(int argc, const char **argv) int num_shards = 0; bool num_shards_specified = false; std::optional bucket_index_max_shards; + int max_concurrent_ios = 32; uint64_t orphan_stale_secs = (24 * 3600); int detail = false; @@ -3916,6 +3922,9 @@ int main(int argc, const char **argv) // do nothing } else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) { // do nothing + } else if (ceph_argparse_binary_flag(args, i, &placement_inline_data, NULL, "--placement-inline-data", (char*)NULL)) { + placement_inline_data_specified = true; + // do nothing } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) { caps = val; } else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) { @@ -6343,6 +6352,9 @@ int main(int argc, const char **argv) if (index_type_specified) { info.index_type = placement_index_type; } + if (placement_inline_data_specified) { + info.inline_data = placement_inline_data; + } ret = check_pool_support_omap(info.get_data_extra_pool()); if (ret < 0) { diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 00d3ed68ed0ec..8a6a157018ef0 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -247,7 +247,12 @@ int AtomicObjectProcessor::prepare(optional_yield y) } if (same_pool) { - head_max_size = max_head_chunk_size; + RGWZonePlacementInfo placement_info; + if (!store->svc()->zone->get_zone_params().get_placement(head_obj->get_bucket()->get_placement_rule().name, &placement_info) || placement_info.inline_data) { + head_max_size = max_head_chunk_size; + } else { + head_max_size = 0; + } chunk_size = max_head_chunk_size; } diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index d886b25a6b4ed..20449c1b9b3e6 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -766,6 +766,7 @@ void RGWZonePlacementInfo::dump(Formatter *f) const encode_json("storage_classes", storage_classes, f); encode_json("data_extra_pool", data_extra_pool, f); encode_json("index_type", (uint32_t)index_type, f); + encode_json("inline_data", inline_data, f); /* no real need for backward compatibility of compression_type and data_pool in here, * rather not clutter the output */ @@ -778,6 +779,7 @@ void RGWZonePlacementInfo::decode_json(JSONObj *obj) JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj); uint32_t it; JSONDecoder::decode_json("index_type", it, obj); + JSONDecoder::decode_json("inline_data", inline_data, obj); index_type = (rgw::BucketIndexType)it; /* backward compatibility, these are now defined in storage_classes */ @@ -1364,3 +1366,4 @@ int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup, } } // namespace rgw + diff --git a/src/rgw/rgw_zone_types.h b/src/rgw/rgw_zone_types.h index abb699f236ebc..f2881dfef926d 100644 --- a/src/rgw/rgw_zone_types.h +++ b/src/rgw/rgw_zone_types.h @@ -217,11 +217,12 @@ struct RGWZonePlacementInfo { rgw_pool data_extra_pool; /* if not set we should use data_pool */ RGWZoneStorageClasses storage_classes; rgw::BucketIndexType index_type; + bool inline_data; - RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal) {} + RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal), inline_data(true) {} void encode(bufferlist& bl) const { - ENCODE_START(7, 1, bl); + ENCODE_START(8, 1, bl); encode(index_pool.to_str(), bl); rgw_pool standard_data_pool = get_data_pool(RGW_STORAGE_CLASS_STANDARD); encode(standard_data_pool.to_str(), bl); @@ -230,11 +231,12 @@ struct RGWZonePlacementInfo { std::string standard_compression_type = get_compression_type(RGW_STORAGE_CLASS_STANDARD); encode(standard_compression_type, bl); encode(storage_classes, bl); + encode(inline_data, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(7, bl); + DECODE_START(8, bl); std::string index_pool_str; std::string data_pool_str; decode(index_pool_str, bl); @@ -261,6 +263,9 @@ struct RGWZonePlacementInfo { storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &standard_data_pool, (!standard_compression_type.empty() ? &standard_compression_type : nullptr)); } + if (struct_v >= 8) { + decode(inline_data, bl); + } DECODE_FINISH(bl); } const rgw_pool& get_data_extra_pool() const { diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 34953ad205456..6c987c413ecb9 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -262,6 +262,9 @@ --data-extra-pool= placement target data extra (non-ec) pool --placement-index-type= placement target index type (normal, indexless, or #id) + --placement-inline-data= + set whether the placement target is configured to store a data + chunk inline in head objects --compression= placement target compression type (plugin name or empty/none) --tier-type= zone tier type --tier-config==[,...] -- 2.39.5