From 28ba9af0e2cac786864a1bc96b1e5654289c72e8 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 4 Nov 2015 11:06:11 -0800 Subject: [PATCH] rgw: update the correct shards when doing full sync The shard we use for each entry (either meta, or data) needs to be selected correctly to match the different logs we keep. Otherwise async notifications don't work correctly Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_bucket.cc | 6 ++++++ src/rgw/rgw_bucket.h | 1 + src/rgw/rgw_data_sync.cc | 4 ++-- src/rgw/rgw_metadata.cc | 15 ++++++++++++++- src/rgw/rgw_metadata.h | 14 +++++++++++++- src/rgw/rgw_sync.cc | 21 ++++++++++++++++++--- 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 6f39e8ad1ee17..cb8e90b990abb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1333,6 +1333,12 @@ void RGWDataChangesLog::update_renewed(rgw_bucket_shard& bs, utime_t& expiration status->cur_expiration = expiration; } +int RGWDataChangesLog::get_log_shard_id(rgw_bucket& bucket, int shard_id) { + rgw_bucket_shard bs(bucket, shard_id); + + return choose_oid(bs); +} + int RGWDataChangesLog::add_entry(rgw_bucket& bucket, int shard_id) { if (!store->need_to_log_data()) return 0; diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 238a07dd1a07f..a5a6763051aa3 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -455,6 +455,7 @@ public: int choose_oid(const rgw_bucket_shard& bs); int add_entry(rgw_bucket& bucket, int shard_id); + int get_log_shard_id(rgw_bucket& bucket, int shard_id); int renew_entries(); int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries, list& entries, diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index 254d69b874517..17e2b3aeb1aa6 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -564,10 +564,10 @@ public: char buf[16]; snprintf(buf, sizeof(buf), ":%d", i); s = key + buf; - yield entries_index->append(s); + yield entries_index->append(s, store->data_log->get_log_shard_id(meta_info.data.get_bucket_info().bucket, i)); } } else { - yield entries_index->append(key); + yield entries_index->append(key, store->data_log->get_log_shard_id(meta_info.data.get_bucket_info().bucket, -1)); } } diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 94dd7e6588985..b961aa761b40a 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -102,6 +102,19 @@ int RGWMetadataLog::add_entry(RGWRados *store, RGWMetadataHandler *handler, cons return store->time_log_add(oid, now, section, key, bl); } +int RGWMetadataLog::get_log_shard_id(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key) +{ + string oid; + + string hash_key; + handler->get_hash_key(section, key, hash_key); + + int shard_id; + store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, &shard_id); + + return shard_id; +} + int RGWMetadataLog::store_entries_in_shard(RGWRados *store, list& entries, int shard_id, librados::AioCompletion *completion) { string oid; @@ -371,7 +384,7 @@ int RGWMetadataManager::register_handler(RGWMetadataHandler *handler) return 0; } -RGWMetadataHandler *RGWMetadataManager::get_handler(const char *type) +RGWMetadataHandler *RGWMetadataManager::get_handler(const string& type) { map::iterator iter = handlers.find(type); if (iter == handlers.end()) diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index 78bb5f9e2ea41..a0bcf463d944d 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -157,6 +157,7 @@ public: RGWMetadataLog(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store), prefix(META_LOG_OBJ_PREFIX), lock("RGWMetaLog::lock") {} int add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl); + int get_log_shard_id(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key); int store_entries_in_shard(RGWRados *store, list& entries, int shard_id, librados::AioCompletion *completion); struct LogListCtx { @@ -238,7 +239,7 @@ public: int register_handler(RGWMetadataHandler *handler); - RGWMetadataHandler *get_handler(const char *type); + RGWMetadataHandler *get_handler(const string& type); int store_md_log_entries(list& entries, int shard_id, librados::AioCompletion *completion); @@ -262,6 +263,17 @@ public: int unlock(string& metadata_key, string& owner_id); RGWMetadataLog *get_log() { return md_log; } + + int get_log_shard_id(const string& section, const string& key, int *shard_id) { + RGWMetadataHandler *handler = get_handler(section); + if (!handler) { + return -EINVAL; + } + + *shard_id = md_log->get_log_shard_id(store, handler, section, key); + + return 0; + } }; #endif diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 69868a6a7e725..165372de13e38 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -540,7 +540,7 @@ class RGWFetchAllMetaCR : public RGWCoroutine { int num_shards; - int req_ret; + int ret_status; list sections; list::iterator sections_iter; @@ -559,7 +559,7 @@ public: RGWFetchAllMetaCR(RGWMetaSyncEnv *_sync_env, int _num_shards, map& _markers) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), num_shards(_num_shards), - req_ret(0), entries_index(NULL), lease_cr(NULL), lost_lock(false), failed(false), markers(_markers) { + ret_status(0), entries_index(NULL), lease_cr(NULL), lost_lock(false), failed(false), markers(_markers) { } ~RGWFetchAllMetaCR() { @@ -644,6 +644,8 @@ public: } iter = result.begin(); for (; iter != result.end(); ++iter) { + RGWRados *store; + int ret; yield { if (!lease_cr->is_locked()) { lost_lock = true; @@ -651,7 +653,15 @@ public: } ldout(cct, 20) << "list metadata: section=" << *sections_iter << " key=" << *iter << dendl; string s = *sections_iter + ":" + *iter; - if (!entries_index->append(s)) { + int shard_id; + store = sync_env->store; + ret = store->meta_mgr->get_log_shard_id(*sections_iter, *iter, &shard_id); + if (ret < 0) { + ldout(cct, 0) << "ERROR: could not determine shard id for " << *sections_iter << ":" << *iter << dendl; + ret_status = ret; + break; + } + if (!entries_index->append(s, shard_id)) { break; } } @@ -689,6 +699,11 @@ public: if (lost_lock) { yield return set_cr_error(-EBUSY); } + + if (ret_status < 0) { + yield return set_cr_error(ret_status); + } + yield return set_cr_done(); } return 0; -- 2.39.5