rgw: log in the same shard for bucket entry point and instance

We'd like to have bucket entry point and instance info at the same
log shard, so that we can process them in order.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 0426bede76c1ffe1ea7883046f662c75633408b5..bd17b25562d86cfbd408feec12acb69f353511b8 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1597,6 +1597,21 @@ public:
     list_keys_info *info = (list_keys_info *)handle;
     delete info;
   }
+
+  /*
+   * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry
+   * point, so that the log entries end up at the same log shard, so that we process them
+   * in order
+   */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    string k;
+    int pos = key.find(':');
+    if (pos < 0)
+      k = key;
+    else
+      k = key.substr(0, pos);
+    hash_key = "bucket:" + k;
+  }
 };
 
 void rgw_bucket_init(RGWMetadataManager *mm)

diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index d27ba14d009c647edbcbb2ddc47191503fb41369..7ace5b317a6fcad0ea768c9d43a9a89b65d5bc7c 100644 (file)
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -74,10 +74,13 @@ struct RGWMetadataLogData {
 WRITE_CLASS_ENCODER(RGWMetadataLogData);
 
 
-int RGWMetadataLog::add_entry(RGWRados *store, const string& section, const string& key, bufferlist& bl) {
+int RGWMetadataLog::add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl) {
   string oid;
 
-  store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, section, key, oid);
+  string hash_key;
+  handler->get_hash_key(section, key, hash_key);
+
+  store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid);
   utime_t now = ceph_clock_now(cct);
   return store->time_log_add(oid, now, section, key, bl);
 }
@@ -494,14 +497,14 @@ int RGWMetadataManager::pre_modify(RGWMetadataHandler *handler, string& section,
   bufferlist logbl;
   ::encode(log_data, logbl);
 
-  int ret = md_log->add_entry(store, section, key, logbl);
+  int ret = md_log->add_entry(store, handler, section, key, logbl);
   if (ret < 0)
     return ret;
 
   return 0;
 }
 
-int RGWMetadataManager::post_modify(const string& section, const string& key, RGWMetadataLogData& log_data,
+int RGWMetadataManager::post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
                                     RGWObjVersionTracker *objv_tracker, int ret)
 {
   if (ret >= 0)
@@ -512,7 +515,7 @@ int RGWMetadataManager::post_modify(const string& section, const string& key, RG
   bufferlist logbl;
   ::encode(log_data, logbl);
 
-  int r = md_log->add_entry(store, section, key, logbl);
+  int r = md_log->add_entry(store, handler, section, key, logbl);
   if (ret < 0)
     return ret;
 
@@ -541,7 +544,7 @@ int RGWMetadataManager::put_entry(RGWMetadataHandler *handler, const string& key
                            objv_tracker, mtime, pattrs);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 
@@ -566,7 +569,7 @@ int RGWMetadataManager::remove_entry(RGWMetadataHandler *handler, string& key, R
   ret = store->delete_obj(NULL, obj);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 
@@ -587,7 +590,7 @@ int RGWMetadataManager::set_attrs(RGWMetadataHandler *handler, string& key,
   ret = store->set_attrs(NULL, obj, attrs, rmattrs, objv_tracker);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 

diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 7c991e852ba55fb26db510b2ea90fdccad03c6f0..43670007cd560c6cda4e6b02574906274d7ab5fd 100644 (file)
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -57,6 +57,11 @@ public:
   virtual int list_keys_init(RGWRados *store, void **phandle) = 0;
   virtual int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) = 0;
   virtual void list_keys_complete(void *handle) = 0;
+
+  /* key to use for hashing entries for log shard placement */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    hash_key = section + ":" + key;
+  }
 };
 
 #define META_LOG_OBJ_PREFIX "meta.log."
@@ -77,7 +82,7 @@ public:
     prefix = META_LOG_OBJ_PREFIX;
   }
 
-  int add_entry(RGWRados *store, const string& section, const string& key, bufferlist& bl);
+  int add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl);
 
   struct LogListCtx {
     int cur_shard;
@@ -117,7 +122,7 @@ class RGWMetadataManager {
   int pre_modify(RGWMetadataHandler *handler, string& section, const string& key,
                  RGWMetadataLogData& log_data, RGWObjVersionTracker *objv_tracker,
                  RGWMDLogStatus op_type);
-  int post_modify(const string& section, const string& key, RGWMetadataLogData& log_data,
+  int post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
                  RGWObjVersionTracker *objv_tracker, int ret);
 
 public: