rgw: add 'inline_data' zone placement info option

Adds a new RGW zone placement info option to control whether
an object's first data chunk is placed in the head object.
This allows admins to make a tradeoff between optimizing for
PUT/GET performance vs. DELETE performance for some cluster
configurations.

Fixes: https://tracker.ceph.com/issues/57965
Signed-off-by: Cory Snyder <csnyder@iland.com>

diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst
index a6609c56dce3c2414077e25d6ef05b44077264a0..54a66e17a0b8af8e1ff60f32aa3aff22a951a5d5 100644 (file)
--- a/doc/man/8/radosgw-admin.rst
+++ b/doc/man/8/radosgw-admin.rst
@@ -743,6 +743,10 @@ Options
 
    The placement target index type (normal, indexless, or #id).
 
+.. option:: --placement-inline-data=<true>
+
+   Whether the placement target is configured to store a data chunk inline in head objects.
+
 .. option:: --tier-type=<type>
 
    The zone tier type.

diff --git a/doc/radosgw/placement.rst b/doc/radosgw/placement.rst
index 595fde2947040ddc74b09d22bd10786cf7adfb9e..6274b022f4a00a5ca6e0837edf3d46ea47c2c30f 100644 (file)
--- a/doc/radosgw/placement.rst
+++ b/doc/radosgw/placement.rst
@@ -86,7 +86,8 @@ The zone placement configuration can be queried with:
                       }
                   },
                   "data_extra_pool": "default.rgw.buckets.non-ec",
-                  "index_type": 0
+                  "index_type": 0,
+                  "inline_data": true
               }
           }
       ],
@@ -123,6 +124,16 @@ Then provide the zone placement info for that target:
         --index-pool default.rgw.temporary.index \
         --data-extra-pool default.rgw.temporary.non-ec
 
+.. note:: With default placement target settings, RGW stores an object's first data chunk in the RADOS "head" object along
+          with xattr metadata. The `--placement-inline-data=false` flag may be passed with the `zone placement add` or
+          `zone placement modify` commands to change this behavior for new objects stored on the target.
+          When data is stored inline (default), it may provide an advantage for read/write workloads since the first chunk of
+          an object's data can be retrieved/stored in a single librados call along with object metadata. On the other hand, a
+          target that does not store data inline can provide a performance benefit for RGW client delete requests when
+          bluestore db is located on faster storage devices (as compared to data devices) since it eliminates the need to access
+          slower devices synchronously while processing the client request. In that case, all data associated with the deleted
+          objects can be removed asynchronously in the background by garbage collection.                                          
+
 .. _adding_a_storage_class:
 
 Adding a Storage Class

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index f18ce8cfbfb78f9b4b25f3c04dcd8d77e9c9a729..46b6ced3cdeda08331c8cc2bc6ec5cbab116253b 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -392,6 +392,9 @@ void usage()
   cout << "   --data-extra-pool=<pool>  placement target data extra (non-ec) pool\n";
   cout << "   --placement-index-type=<type>\n";
   cout << "                             placement target index type (normal, indexless, or #id)\n";
+  cout << "   --placement-inline-data=<true>\n";
+  cout << "                             set whether the placement target is configured to store a data\n";
+  cout << "                             chunk inline in head objects\n";
   cout << "   --compression=<type>      placement target compression type (plugin name or empty/none)\n";
   cout << "   --tier-type=<type>        zone tier type\n";
   cout << "   --tier-config=<k>=<v>[,...]\n";
@@ -3537,6 +3540,8 @@ int main(int argc, const char **argv)
   list<string> tags;
   list<string> tags_add;
   list<string> tags_rm;
+  int placement_inline_data = true;
+  bool placement_inline_data_specified = false;
 
   int64_t max_objects = -1;
   int64_t max_size = -1;
@@ -3574,6 +3579,7 @@ int main(int argc, const char **argv)
   int num_shards = 0;
   bool num_shards_specified = false;
   std::optional<int> bucket_index_max_shards;
+
   int max_concurrent_ios = 32;
   uint64_t orphan_stale_secs = (24 * 3600);
   int detail = false;
@@ -3916,6 +3922,9 @@ int main(int argc, const char **argv)
      // do nothing
     } else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) {
      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &placement_inline_data, NULL, "--placement-inline-data", (char*)NULL)) {
+      placement_inline_data_specified = true;
+     // do nothing
     } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
       caps = val;
     } else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) {
@@ -6343,6 +6352,9 @@ int main(int argc, const char **argv)
           if (index_type_specified) {
            info.index_type = placement_index_type;
           }
+          if (placement_inline_data_specified) {
+            info.inline_data = placement_inline_data;
+          }
 
           ret = check_pool_support_omap(info.get_data_extra_pool());
           if (ret < 0) {

diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc
index 00d3ed68ed0ecf7f6f4bc5cb27a201c053d0b458..8a6a157018ef024a0bce8d56a94ea55e40fc3716 100644 (file)
--- a/src/rgw/rgw_putobj_processor.cc
+++ b/src/rgw/rgw_putobj_processor.cc
@@ -247,7 +247,12 @@ int AtomicObjectProcessor::prepare(optional_yield y)
   }
 
   if (same_pool) {
-    head_max_size = max_head_chunk_size;
+    RGWZonePlacementInfo placement_info;
+    if (!store->svc()->zone->get_zone_params().get_placement(head_obj->get_bucket()->get_placement_rule().name, &placement_info) || placement_info.inline_data) {
+      head_max_size = max_head_chunk_size;
+    } else {
+      head_max_size = 0;
+    }
     chunk_size = max_head_chunk_size;
   }
 

diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc
index d886b25a6b4edd1421e17e9f83eda8b2bb12deba..20449c1b9b3e6ab15cbfa2a40da3bbde82cd39f4 100644 (file)
--- a/src/rgw/rgw_zone.cc
+++ b/src/rgw/rgw_zone.cc
@@ -766,6 +766,7 @@ void RGWZonePlacementInfo::dump(Formatter *f) const
   encode_json("storage_classes", storage_classes, f);
   encode_json("data_extra_pool", data_extra_pool, f);
   encode_json("index_type", (uint32_t)index_type, f);
+  encode_json("inline_data", inline_data, f);
 
   /* no real need for backward compatibility of compression_type and data_pool in here,
    * rather not clutter the output */
@@ -778,6 +779,7 @@ void RGWZonePlacementInfo::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj);
   uint32_t it;
   JSONDecoder::decode_json("index_type", it, obj);
+  JSONDecoder::decode_json("inline_data", inline_data, obj);
   index_type = (rgw::BucketIndexType)it;
 
   /* backward compatibility, these are now defined in storage_classes */
@@ -1364,3 +1366,4 @@ int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup,
 }
 
 } // namespace rgw
+

diff --git a/src/rgw/rgw_zone_types.h b/src/rgw/rgw_zone_types.h
index abb699f236ebc9dcd49e508767b43593a774be5c..f2881dfef926d875ef11fe677c4bea892de40896 100644 (file)
--- a/src/rgw/rgw_zone_types.h
+++ b/src/rgw/rgw_zone_types.h
@@ -217,11 +217,12 @@ struct RGWZonePlacementInfo {
   rgw_pool data_extra_pool; /* if not set we should use data_pool */
   RGWZoneStorageClasses storage_classes;
   rgw::BucketIndexType index_type;
+  bool inline_data;
 
-  RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal) {}
+  RGWZonePlacementInfo() : index_type(rgw::BucketIndexType::Normal), inline_data(true) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(7, 1, bl);
+    ENCODE_START(8, 1, bl);
     encode(index_pool.to_str(), bl);
     rgw_pool standard_data_pool = get_data_pool(RGW_STORAGE_CLASS_STANDARD);
     encode(standard_data_pool.to_str(), bl);
@@ -230,11 +231,12 @@ struct RGWZonePlacementInfo {
     std::string standard_compression_type = get_compression_type(RGW_STORAGE_CLASS_STANDARD);
     encode(standard_compression_type, bl);
     encode(storage_classes, bl);
+    encode(inline_data, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(7, bl);
+    DECODE_START(8, bl);
     std::string index_pool_str;
     std::string data_pool_str;
     decode(index_pool_str, bl);
@@ -261,6 +263,9 @@ struct RGWZonePlacementInfo {
       storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &standard_data_pool,
                                         (!standard_compression_type.empty() ? &standard_compression_type : nullptr));
     }
+    if (struct_v >= 8) {
+      decode(inline_data, bl);
+    }
     DECODE_FINISH(bl);
   }
   const rgw_pool& get_data_extra_pool() const {

diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 34953ad205456ef8b055e87902aa958d8d02ca36..6c987c413ecb948a2ab51a10c7e640bbcf6721c4 100644 (file)
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -262,6 +262,9 @@
      --data-extra-pool=<pool>  placement target data extra (non-ec) pool
      --placement-index-type=<type>
                                placement target index type (normal, indexless, or #id)
+     --placement-inline-data=<true>
+                               set whether the placement target is configured to store a data
+                               chunk inline in head objects
      --compression=<type>      placement target compression type (plugin name or empty/none)
      --tier-type=<type>        zone tier type
      --tier-config=<k>=<v>[,...]