From: Yehuda Sadeh Date: Mon, 24 Nov 2014 22:30:20 +0000 (-0800) Subject: rgw: bi log list/trim can get specific bucket shard X-Git-Tag: v0.92~12^2~25 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=381f68acdec8c2e55313c3509f2fed8510337cfa;p=ceph.git rgw: bi log list/trim can get specific bucket shard bucket shard can be specified on the bucket instance param. It can be added like this: [:shard-id] Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 9cf4a734c33e..d0f9f5843dc7 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -2350,7 +2350,7 @@ next: do { list entries; - ret = store->list_bi_log_entries(bucket, marker, max_entries - count, entries, &truncated); + ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated); if (ret < 0) { cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl; return -ret; @@ -2382,7 +2382,7 @@ next: cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; return -ret; } - ret = store->trim_bi_log_entries(bucket, start_marker, end_marker); + ret = store->trim_bi_log_entries(bucket, shard_id, start_marker, end_marker); if (ret < 0) { cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl; return -ret; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 83e451378b4a..b8c2778b0ec5 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -233,6 +233,32 @@ int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersi return store->meta_mgr->remove_entry(bucket_instance_meta_handler, entry, objv_tracker); } +int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *target_bucket_instance, int *shard_id) +{ + ssize_t pos = bucket_instance.rfind(':'); + if (pos < 0) { + return -EINVAL; + } + + string first = bucket_instance.substr(0, pos); + string second = bucket_instance.substr(pos + 1); + + if (first.find(':') == string::npos) { + *shard_id = -1; + *target_bucket_instance = bucket_instance; + return 0; + } + + *target_bucket_instance = first; + string err; + *shard_id = strict_strtol(second.c_str(), 10, &err); + if (!err.empty()) { + return -EINVAL; + } + + return 0; +} + int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info, map& attrs, map* rmattrs, diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 3bdd68c057d0..d53da2f9ba63 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -32,6 +32,8 @@ extern int rgw_bucket_instance_store_info(RGWRados *store, string& oid, bufferli map *pattrs, RGWObjVersionTracker *objv_tracker, time_t mtime); +extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *target_bucket_instance, int *shard_id); + extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker); extern int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index a485954e2d53..6cf4aaa9e85b 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3808,7 +3808,7 @@ int RGWRados::open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_ } int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - vector& bucket_objs) { + vector& bucket_objs, int shard_id) { string bucket_oid_base; int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base); if (ret < 0) @@ -3820,16 +3820,16 @@ int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, if (ret < 0) return ret; - get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs); + get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs, shard_id); return 0; } template int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - map& bucket_objs) + map& bucket_objs, int shard_id) { vector oids; - int ret = open_bucket_index(bucket, index_ctx, oids); + int ret = open_bucket_index(bucket, index_ctx, oids, shard_id); if (ret < 0) return ret; @@ -6101,7 +6101,7 @@ int RGWRados::list_raw_objects(rgw_bucket& pool, const string& prefix_filter, return oids.size(); } -int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max, +int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max, std::list& result, bool *truncated) { ldout(cct, 20) << __func__ << bucket << " marker " << marker << " max " << max << dendl; @@ -6109,12 +6109,12 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t m librados::IoCtx index_ctx; map bi_log_lists; - int r = open_bucket_index(bucket, index_ctx, bi_log_lists); + int r = open_bucket_index(bucket, index_ctx, bi_log_lists, shard_id); if (r < 0) return r; BucketIndexShardsManager marker_mgr; - bool has_shards = (bi_log_lists.size() > 1); + bool has_shards = (bi_log_lists.size() > 1 || shard_id >= 0); // If there are multiple shards for the bucket index object, the marker // should have the pattern '{shard_oid_1}#{shard_marker_1},{shard_oid_2}# // {shard_marker_2}...', if there is no sharding, the bi_log_list should @@ -6183,15 +6183,15 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t m return 0; } -int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, string& start_marker, string& end_marker) +int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& start_marker, string& end_marker) { librados::IoCtx index_ctx; vector bucket_objs; - int r = open_bucket_index(bucket, index_ctx, bucket_objs); + int r = open_bucket_index(bucket, index_ctx, bucket_objs, shard_id); if (r < 0) return r; - bool has_shards = bucket_objs.size() > 1; + bool has_shards = bucket_objs.size() > 1 || shard_id >= 0; BucketIndexShardsManager start_marker_mgr; r = start_marker_mgr.from_string(start_marker, has_shards, bucket_objs.front()); if (r < 0) @@ -6950,14 +6950,22 @@ int RGWRados::remove_temp_objects(string date, string time) } void RGWRados::get_bucket_index_objects(const string& bucket_oid_base, - const uint32_t num_shards, vector& bucket_objects) + uint32_t num_shards, vector& bucket_objects, int shard_id) { if (!num_shards) { bucket_objects.push_back(bucket_oid_base); } else { char buf[bucket_oid_base.size() + 32]; - for (uint32_t i = 0; i < num_shards; ++i) { - snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i); + if (shard_id < 0) { + for (uint32_t i = 0; i < num_shards; ++i) { + snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i); + bucket_objects.push_back(string(buf)); + } + } else { + if ((uint32_t)shard_id > num_shards) { + return; + } + snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id); bucket_objects.push_back(string(buf)); } } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 819f494925bd..d8f6c168a0f8 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1257,10 +1257,10 @@ class RGWRados int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx, const string& obj_key, string *bucket_obj); int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - vector& bucket_objs); + vector& bucket_objs, int shard_id = -1); template int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, - map& bucket_objs); + map& bucket_objs, int shard_id = -1); void build_bucket_index_marker(const string& shard_name, const string& shard_marker, string *marker); @@ -1874,8 +1874,8 @@ public: return cls_obj_complete_cancel(bucket, tag, oid.object, oid.get_hash_object()); } - int list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max, std::list& result, bool *truncated); - int trim_bi_log_entries(rgw_bucket& bucket, string& marker, string& end_marker); + int list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max, std::list& result, bool *truncated); + int trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, string& end_marker); int cls_obj_usage_log_add(const string& oid, rgw_usage_log_info& info); int cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, @@ -1960,8 +1960,8 @@ public: * num_shards [in] - number of bucket index object shards. * bucket_objs [out] - filled by this method, a list of bucket index objects. */ - void get_bucket_index_objects(const string& bucket_oid_base, const uint32_t num_shards, - vector& bucket_objs); + void get_bucket_index_objects(const string& bucket_oid_base, uint32_t num_shards, + vector& bucket_objs, int shard_id = -1); /** * Get the bucket index object with the given base bucket index object and object key, diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index 9f32fc9ebd3f..1db6cadb0d8e 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -282,6 +282,12 @@ void RGWOp_BILog_List::execute() { return; } + int shard_id; + http_ret = rgw_bucket_parse_bucket_instance(bucket_instance, &bucket_instance, &shard_id); + if (http_ret < 0) { + return; + } + if (!bucket_instance.empty()) { http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL); if (http_ret < 0) { @@ -307,7 +313,7 @@ void RGWOp_BILog_List::execute() { send_response(); do { list entries; - int ret = store->list_bi_log_entries(bucket_info.bucket, + int ret = store->list_bi_log_entries(bucket_info.bucket, shard_id, marker, max_entries - count, entries, &truncated); if (ret < 0) { @@ -419,6 +425,13 @@ void RGWOp_BILog_Delete::execute() { http_ret = -EINVAL; return; } + + int shard_id; + http_ret = rgw_bucket_parse_bucket_instance(bucket_instance, &bucket_instance, &shard_id); + if (http_ret < 0) { + return; + } + if (!bucket_instance.empty()) { http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL); if (http_ret < 0) { @@ -432,7 +445,7 @@ void RGWOp_BILog_Delete::execute() { return; } } - http_ret = store->trim_bi_log_entries(bucket_info.bucket, start_marker, end_marker); + http_ret = store->trim_bi_log_entries(bucket_info.bucket, shard_id, start_marker, end_marker); if (http_ret < 0) { dout(5) << "ERROR: trim_bi_log_entries() " << dendl; }