rgw: increase max shard prime, consolidate logic

Handle larger num of shards (max ~64k now). Use the same hash logic for shard ids
everywhere.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 6db1405282b56f1e6f9370f8c9a0aaa470476417..094a6c1b1a38a83acb49647da8cd4b244224537c 100644 (file)
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -20,15 +20,11 @@ static string gc_oid_prefix = "gc";
 static string gc_index_lock_name = "gc_process";
 
 
-#define HASH_PRIME 7877
-
 void RGWGC::initialize(CephContext *_cct, RGWRados *_store) {
   cct = _cct;
   store = _store;
 
-  max_objs = cct->_conf->rgw_gc_max_objs;
-  if (max_objs > HASH_PRIME)
-    max_objs = HASH_PRIME;
+  max_objs = min(cct->_conf->rgw_gc_max_objs, rgw_shards_max());
 
   obj_names = new string[max_objs];
 
@@ -47,7 +43,7 @@ void RGWGC::finalize()
 
 int RGWGC::tag_index(const string& tag)
 {
-  return ceph_str_hash_linux(tag.c_str(), tag.size()) % HASH_PRIME % max_objs;
+  return rgw_shards_hash(tag, max_objs);
 }
 
 void RGWGC::add_chain(ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index abc77f30ff11be67baa34c2171be83af759c2000..b86a08c42783236eb80ef7e3f824f5ae3e19d88a 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -4538,10 +4538,10 @@ int RGWRados::init_complete()
 
   bucket_index_max_shards = (cct->_conf->rgw_override_bucket_index_max_shards ? cct->_conf->rgw_override_bucket_index_max_shards :
                              get_zone().bucket_index_max_shards);
-  if (bucket_index_max_shards > MAX_BUCKET_INDEX_SHARDS_PRIME) {
-    bucket_index_max_shards = MAX_BUCKET_INDEX_SHARDS_PRIME;
+  if (bucket_index_max_shards > get_max_bucket_shards()) {
+    bucket_index_max_shards = get_max_bucket_shards();
     ldout(cct, 1) << __func__ << " bucket index max shards is too large, reset to value: "
-      << MAX_BUCKET_INDEX_SHARDS_PRIME << dendl;
+      << get_max_bucket_shards() << dendl;
   }
   ldout(cct, 20) << __func__ << " bucket index max shards: " << bucket_index_max_shards << dendl;
 
@@ -5117,12 +5117,9 @@ next:
   return 0;
 }
 
-#define MAX_SHARDS_PRIME 7877
-
 int RGWRados::key_to_shard_id(const string& key, int max_shards)
 {
-  uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()) % MAX_SHARDS_PRIME;
-  return val % max_shards;
+  return rgw_shards_hash(key, max_shards);
 }
 
 void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id)
@@ -5299,16 +5296,14 @@ string RGWRados::objexp_hint_get_shardname(int shard_num)
   return objname + buf;
 }
 
-#define MAX_OBJEXP_SHARDS_PRIME 7877
-
 int RGWRados::objexp_key_shard(const rgw_obj_index_key& key)
 {
   string obj_key = key.name + key.instance;
   int num_shards = cct->_conf->rgw_objexp_hints_num_shards;
   uint32_t sid = ceph_str_hash_linux(obj_key.c_str(), obj_key.size());
   uint32_t sid2 = sid ^ ((sid & 0xFF) << 24);
-  sid = sid2 % MAX_OBJEXP_SHARDS_PRIME % num_shards;
-  return sid % num_shards;
+  sid = rgw_shards_mod(sid2, num_shards);
+  return sid;
 }
 
 static string objexp_hint_get_keyext(const string& tenant_name,
@@ -13216,7 +13211,7 @@ int RGWRados::get_target_shard_id(const RGWBucketInfo& bucket_info, const string
       } else {
         uint32_t sid = ceph_str_hash_linux(obj_key.c_str(), obj_key.size());
         uint32_t sid2 = sid ^ ((sid & 0xFF) << 24);
-        sid = sid2 % MAX_BUCKET_INDEX_SHARDS_PRIME % bucket_info.num_shards;
+        sid = rgw_shards_mod(sid2, bucket_info.num_shards);
         if (shard_id) {
           *shard_id = (int)sid;
         }
@@ -13256,7 +13251,7 @@ int RGWRados::get_bucket_index_object(const string& bucket_oid_base, const strin
       } else {
         uint32_t sid = ceph_str_hash_linux(obj_key.c_str(), obj_key.size());
         uint32_t sid2 = sid ^ ((sid & 0xFF) << 24);
-        sid = sid2 % MAX_BUCKET_INDEX_SHARDS_PRIME % num_shards;
+        sid = rgw_shards_mod(sid2, num_shards);
         char buf[bucket_oid_base.size() + 32];
         snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), sid);
         (*bucket_obj) = buf;

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index ce8eac5108865c7468ca50b125c65ab6a375d363..ca476a2c207fab20a7ffdcc286ff0b42173d4017 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -53,7 +53,26 @@ class RGWReshardWait;
 
 #define RGW_NO_SHARD -1
 
-#define MAX_BUCKET_INDEX_SHARDS_PRIME 7877
+#define RGW_SHARDS_PRIME_0 7877
+#define RGW_SHARDS_PRIME_1 65521
+
+static inline int rgw_shards_mod(unsigned hval, int max_shards)
+{
+  if (max_shards <= RGW_SHARDS_PRIME_0) {
+    return hval % RGW_SHARDS_PRIME_0 % max_shards;
+  }
+  return hval % RGW_SHARDS_PRIME_1 % max_shards;
+}
+
+static inline int rgw_shards_hash(const string& key, int max_shards)
+{
+  return rgw_shards_mod(ceph_str_hash_linux(key.c_str(), key.size()), max_shards);
+}
+
+static inline int rgw_shards_max()
+{
+  return RGW_SHARDS_PRIME_1;
+}
 
 static inline void prepend_bucket_marker(const rgw_bucket& bucket, const string& orig_oid, string& oid)
 {
@@ -2478,7 +2497,7 @@ public:
   int get_max_chunk_size(const string& placement_rule, const rgw_obj& obj, uint64_t *max_chunk_size);
 
   uint32_t get_max_bucket_shards() {
-    return MAX_BUCKET_INDEX_SHARDS_PRIME;
+    return rgw_shards_max();
   }
 
   int get_raw_obj_ref(const rgw_raw_obj& obj, rgw_rados_ref *ref, rgw_pool *pool = NULL);