From: Yehuda Sadeh Date: Fri, 5 Dec 2014 01:15:15 +0000 (-0800) Subject: rgw: modify bucket instance shard marker ids X-Git-Tag: v0.92~12^2~17 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a063cbac8d1980d06125fe4bb5836f5b6180ccb3;p=ceph.git rgw: modify bucket instance shard marker ids Instead of having the markers prefixed by the oids, use the bucket instance id. Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 444b0e468e03..27608572c522 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3834,7 +3834,7 @@ int RGWRados::open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_ } int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - vector& bucket_objs, int shard_id) { + vector& bucket_objs, int shard_id, vector *bucket_instance_ids) { string bucket_oid_base; int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base); if (ret < 0) @@ -3847,15 +3847,18 @@ int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, return ret; get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs, shard_id); + if (bucket_instance_ids) { + get_bucket_instance_ids(binfo, shard_id, bucket_instance_ids); + } return 0; } template int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - map& bucket_objs, int shard_id) + map& bucket_objs, int shard_id, vector *bucket_instance_ids) { vector oids; - int ret = open_bucket_index(bucket, index_ctx, oids, shard_id); + int ret = open_bucket_index(bucket, index_ctx, oids, shard_id, bucket_instance_ids); if (ret < 0) return ret; @@ -5522,22 +5525,26 @@ int RGWRados::get_bucket_stats(rgw_bucket& bucket, string *bucket_ver, string *m map& stats, string *max_marker) { map headers; - int r = cls_bucket_head(bucket, headers); + vector bucket_instance_ids; + int r = cls_bucket_head(bucket, headers, &bucket_instance_ids); if (r < 0) return r; + assert(headers.size() == bucket_instance_ids.size()); + map::iterator iter = headers.begin(); + vector::iterator viter = bucket_instance_ids.begin(); BucketIndexShardsManager ver_mgr; BucketIndexShardsManager master_ver_mgr; BucketIndexShardsManager marker_mgr; char buf[64]; - for(; iter != headers.end(); ++iter) { + for(; iter != headers.end(); ++iter, ++viter) { accumulate_raw_stats(iter->second, stats); snprintf(buf, sizeof(buf), "%lu", iter->second.ver); - ver_mgr.add(iter->first, string(buf)); + ver_mgr.add(*viter, string(buf)); snprintf(buf, sizeof(buf), "%lu", iter->second.master_ver); - master_ver_mgr.add(iter->first, string(buf)); - marker_mgr.add(iter->first, iter->second.max_marker); + master_ver_mgr.add(*viter, string(buf)); + marker_mgr.add(*viter, iter->second.max_marker); } ver_mgr.to_string(bucket_ver); master_ver_mgr.to_string(master_ver); @@ -6156,7 +6163,8 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& mark librados::IoCtx index_ctx; map bi_log_lists; - int r = open_bucket_index(bucket, index_ctx, bi_log_lists, shard_id); + vector bucket_instance_ids; + int r = open_bucket_index(bucket, index_ctx, bi_log_lists, shard_id, &bucket_instance_ids); if (r < 0) return r; @@ -6176,13 +6184,11 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& mark vector::iterator> vcurrents; vector::iterator> vends; - vector vnames; if (truncated) { *truncated = false; } map::iterator miter = bi_log_lists.begin(); for (; miter != bi_log_lists.end(); ++miter) { - vnames.push_back(miter->first); vcurrents.push_back(miter->second.entries.begin()); vends.push_back(miter->second.entries.end()); if (truncated) { @@ -6198,14 +6204,15 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& mark ++vcurrents[i], ++i) { if (vcurrents[i] != vends[i]) { rgw_bi_log_entry& entry = *(vcurrents[i]); + string& name = bucket_instance_ids[i]; if (has_shards) { // Put the shard name as part of the ID, so that caller can easy find out // the next marker string tmp_id; - build_bucket_index_marker(vnames[i], entry.id, &tmp_id); + build_bucket_index_marker(name, entry.id, &tmp_id); entry.id.swap(tmp_id); } - marker_mgr.add(vnames[i], entry.id); + marker_mgr.add(name, entry.id); result.push_back(entry); has_more = true; } @@ -6656,11 +6663,11 @@ int RGWRados::check_disk_state(librados::IoCtx io_ctx, return 0; } -int RGWRados::cls_bucket_head(rgw_bucket& bucket, map& headers) +int RGWRados::cls_bucket_head(rgw_bucket& bucket, map& headers, vector *bucket_instance_ids) { librados::IoCtx index_ctx; map list_results; - int r = open_bucket_index(bucket, index_ctx, list_results); + int r = open_bucket_index(bucket, index_ctx, list_results, -1, bucket_instance_ids); if (r < 0) return r; @@ -7004,6 +7011,29 @@ void RGWRados::get_bucket_index_objects(const string& bucket_oid_base, } } +void RGWRados::get_bucket_instance_ids(RGWBucketInfo& bucket_info, int shard_id, vector *result) +{ + rgw_bucket& bucket = bucket_info.bucket; + string plain_id = bucket.name + ":" + bucket.bucket_id; + if (!bucket_info.num_shards) { + result->push_back(plain_id); + } else { + char buf[16]; + if (shard_id < 0) { + for (uint32_t i = 0; i < bucket_info.num_shards; ++i) { + snprintf(buf, sizeof(buf), ":%d", i); + result->push_back(plain_id + buf); + } + } else { + if ((uint32_t)shard_id > bucket_info.num_shards) { + return; + } + snprintf(buf, sizeof(buf), ":%d", shard_id); + result->push_back(plain_id + buf); + } + } +} + int RGWRados::get_bucket_index_object(const string& bucket_oid_base, const string& obj_key, uint32_t num_shards, RGWBucketInfo::BIShardsHashType hash_type, string *bucket_obj, int *shard_id) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 1873ed47e583..c5376b779bed 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1257,13 +1257,15 @@ class RGWRados int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx, const string& obj_key, string *bucket_obj, int *shard_id); int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - vector& bucket_objs, int shard_id = -1); + vector& bucket_objs, int shard_id = -1, vector *bucket_instance_ids = NULL); template int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - map& bucket_objs, int shard_id = -1); + map& bucket_objs, int shard_id = -1, vector *bucket_instance_ids = NULL); void build_bucket_index_marker(const string& shard_name, const string& shard_marker, string *marker); + void get_bucket_instance_ids(RGWBucketInfo& bucket_info, int shard_id, vector *result); + struct GetObjState { librados::IoCtx io_ctx; bool sent_data; @@ -1865,7 +1867,7 @@ public: int cls_bucket_list(rgw_bucket& bucket, const string& start, const string& prefix, uint32_t hint_num, map& m, bool *is_truncated, string *last_entry, bool (*force_check_filter)(const string& name) = NULL); - int cls_bucket_head(rgw_bucket& bucket, map& headers); + int cls_bucket_head(rgw_bucket& bucket, map& headers, vector *bucket_instance_ids = NULL); int cls_bucket_head_async(rgw_bucket& bucket, RGWGetDirHeader_CB *ctx, int *num_aio); int prepare_update_index(RGWObjState *state, BucketShard& bucket_shard,