rgw/dedup: extend shard tiers, add overflow guard, and fix bucket filter

  The dedup subsystem was limited to 512 MD5 shards with 10% headroom,
  capping effective operation at ~4 billion objects.
  - Raise MAX_MD5_SHARD from 512 to 2048 and add new shard tiers for
    1024 (2GB), 2048 (4GB) shards, supporting up to ~213 billion objects.
  - Increase headroom from 10% to 20% to improve hash table load factor.
  - Reject pools that exceed the maximum capacity with -EOVERFLOW
  - Apply bucket filter in collect_all_buckets_stats() so that filtered
    buckets do not inflate the object count used for shard sizing.
  - Update sizing table, comments, and documentation to reflect the new
    tiers and capacity limit.

Signed-off-by: benhanokh <gbenhano@redhat.com>

diff --git a/doc/radosgw/s3_objects_dedup.rst b/doc/radosgw/s3_objects_dedup.rst
index 132a02319b748ffa6ab88c9504c4b076b5e0fdc6..3c731766b76ba053edec194964464057457e7e6f 100644 (file)
--- a/doc/radosgw/s3_objects_dedup.rst
+++ b/doc/radosgw/s3_objects_dedup.rst
@@ -158,19 +158,27 @@ Memory Usage
  +------------------+----------+
  | RGW Object Count |  Memory  |
  +==================+==========+
- | 1M               | 8 MB     |
+ |      1M          |    8 MB  |
  +------------------+----------+
- | 4M               | 16 MB    |
+ |      4M          |   16 MB  |
  +------------------+----------+
- | 16M              | 32 MB    |
+ |     16M          |   32 MB  |
  +------------------+----------+
- | 64M              | 64 MB    |
+ |     64M          |   64 MB  |
  +------------------+----------+
- | 256M             | 128 MB   |
+ |    256M          |  128 MB  |
  +------------------+----------+
- | 1024M (1G)       | 256 MB   |
+ |   1024M   (1G)   |  256 MB  |
  +------------------+----------+
- | 4096M (4G)       | 512 MB   |
+ |   4096M   (4G)   |  512 MB  |
  +------------------+----------+
- | 16384M (16G)     | 1024 MB  |
+ |  16384M  (16G)   | 1024 MB  |
  +------------------+----------+
+ |  65536M  (64G)   | 2048 MB  |
+ +------------------+----------+
+ | 262144M (256G)   | 4096 MB  |
+ +------------------+----------+
+
+ .. note::
+     Pools with more than ~213 billion user objects (256B with headroom) exceed the
+     dedup system's maximum capacity and will be rejected at startup.

diff --git a/src/rgw/driver/rados/rgw_dedup.cc b/src/rgw/driver/rados/rgw_dedup.cc
index 260a8315f5e40f0445d8f546cc31d106d12934c3..d9551159a4876eb923c5808e0a239cc2bc6551bc 100644 (file)
--- a/src/rgw/driver/rados/rgw_dedup.cc
+++ b/src/rgw/driver/rados/rgw_dedup.cc
@@ -2681,6 +2681,11 @@ namespace rgw::dedup {
             goto err;
           }
           ldpp_dout(dpp, 20) <<__func__ << "::bucket=" << bucket << dendl;
+          if (!d_filter.allow_bucket(bucket.name)) {
+            ldpp_dout(dpp, 10) << __func__ << "::skip bucket (filter): "
+                               << bucket.name << dendl;
+            continue;
+          }
           ret = read_bucket_stats(bucket, &d_all_buckets_obj_count,
                                   &d_all_buckets_obj_size);
           if (unlikely(ret != 0)) {
@@ -2944,14 +2949,22 @@ namespace rgw::dedup {
   //-------------------------------------------------------------------------------
   //    64M      ||     32       ||    64MB ||   64MB/32 =  2.00M *  32 =    64M ||
   //   256M      ||     64       ||   128MB ||  128MB/32 =  4.00M *  64 =   256M ||
-  //  1024M( 1G) ||    128       ||   256MB ||  256MB/32 =  8.00M * 128 =  1024M ||
-  //  4096M( 4G) ||    256       ||   512MB ||  512MB/32 = 16M.00 * 256 =  4096M ||
-  // 16384M(16G) ||    512       ||  1024MB || 1024MB/32 = 32M.00 * 512 = 16384M ||
+  //  1024M(  1G)||    128       ||   256MB ||  256MB/32 =  8.00M * 128 =  1024M ||
+  //  4096M(  4G)||    256       ||   512MB ||  512MB/32 = 16M.00 * 256 =  4096M ||
+  //-------------------------------------------------------------------------------
+  // 16384M( 16G)||    512       ||  1024MB || 1024MB/32 = 32M.00 * 512 = 16384M ||
+  // 65536M( 64G)||   1024       ||  2048MB || 2048MB/32 = 64M.00 *1024 = 65536M ||
+  //262144M(256G)||   2048       ||  4096MB || 4096MB/32 =128M.00 *2048 =262144M ||
   //-------------||--------------||---------||-----------------------------------||
+  //   > 256G    ||  REJECTED    ||   N/A   || Pool exceeds max supported size   ||
+  //=============||==============||=========||===================================||
+
   static md5_shard_t calc_num_md5_shards(uint64_t obj_count)
   {
-    // create headroom by allocating space for a 10% bigger system
-    obj_count = obj_count + (obj_count/10);
+    // create headroom by allocating space for a 20% bigger system
+    // NOTE: all thresholds below compare against the inflated obj_count,
+    // not the raw count reported by pool stats.
+    obj_count = obj_count + (obj_count/5);
 
     uint64_t M = 1024 * 1024;
     if (obj_count < 1*M) {
@@ -2979,12 +2992,26 @@ namespace rgw::dedup {
       return 128;
     }
     else if (obj_count < 4*1024*M) {
-      // less than 4096M objects -> use 256 shards (512MB)
+      // less than   4B objects -> use 256 shards (512MB)
       return 256;
     }
-    else {
+    else if (obj_count < 16*1024*M) {
+      // less than  16B objects -> use 512 shards (1GB)
       return 512;
     }
+    else if (obj_count < 64*1024*M) {
+      // less than  64B objects -> use 1024 shards (2GB)
+      return 1024;
+    }
+    else if (obj_count < 256*1024*M) {
+      // less than 256B objects -> use 2048 shards (4GB)
+      return 2048;
+    }
+    else {
+      // Dedup supports systems with up-to ~213 billion user objects per-pool
+      // With headroom that translates to about 256B objects
+      return MD5_SHARD_HARD_LIMIT + 1;
+    }
   }
 
   //---------------------------------------------------------------------------
@@ -2996,10 +3023,17 @@ namespace rgw::dedup {
     }
 
     md5_shard_t num_md5_shards = calc_num_md5_shards(d_all_buckets_obj_count);
+    if (num_md5_shards > MD5_SHARD_HARD_LIMIT) {
+      derr << __func__ << "::Pool has too many objects: ("
+           << d_all_buckets_obj_count << ") Max supported is 213 billion" << dendl;
+      return -EOVERFLOW;
+    }
+
     num_md5_shards = std::min(num_md5_shards, MAX_MD5_SHARD);
     num_md5_shards = std::max(num_md5_shards, MIN_MD5_SHARD);
     work_shard_t num_work_shards = num_md5_shards;
     num_work_shards = std::min(num_work_shards, MAX_WORK_SHARD);
+    num_work_shards = std::max(num_work_shards, MIN_WORK_SHARD);
 
     ldpp_dout(dpp, 5) << __func__ << "::obj_count=" <<d_all_buckets_obj_count
                       << "::num_md5_shards=" << num_md5_shards
@@ -3007,7 +3041,7 @@ namespace rgw::dedup {
     // init handles and create the dedup_pool
     ret = init_rados_access_handles(true);
     if (ret != 0) {
-      derr << "dedup_bg::resume() failed init_rados_access_handles() ret="
+      derr << __func__ << "::failed init_rados_access_handles() ret="
            << ret << "::" << cpp_strerror(-ret) << dendl;
       return ret;
     }

diff --git a/src/rgw/driver/rados/rgw_dedup_utils.h b/src/rgw/driver/rados/rgw_dedup_utils.h
index 1159c130c6055e73596ea2e8f86c66a04640e30e..a09786bf72dbc541bf6aa7dbb3cd8356bbd512bc 100644 (file)
--- a/src/rgw/driver/rados/rgw_dedup_utils.h
+++ b/src/rgw/driver/rados/rgw_dedup_utils.h
@@ -38,7 +38,7 @@ namespace rgw::dedup {
 
   // Those are the correct values for production system
   const work_shard_t MAX_WORK_SHARD = 255;
-  const md5_shard_t  MAX_MD5_SHARD  = 512;
+  const md5_shard_t  MAX_MD5_SHARD  = 2048;
 
   const work_shard_t NULL_WORK_SHARD = 0xFFFF;
   const md5_shard_t  NULL_MD5_SHARD  = 0xFFFF;