From: Yehuda Sadeh Date: Mon, 27 Jan 2014 19:27:56 +0000 (-0800) Subject: Merge branch 'wip-user-quota-2' X-Git-Tag: v0.78~270 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cacdfd9165d32652bf82d3400c15485a89824a3d;p=ceph.git Merge branch 'wip-user-quota-2' Conflicts: src/common/config_opts.h src/rgw/rgw_common.h src/rgw/rgw_json_enc.cc src/rgw/rgw_quota.cc src/rgw/rgw_rados.cc src/rgw/rgw_rados.h src/rgw/rgw_user.cc src/rgw/rgw_user.h Reviewed-by: Josh Durgin Signed-off-by: Yehuda Sadeh --- cacdfd9165d32652bf82d3400c15485a89824a3d diff --cc src/cls/Makefile.am index f49d49def49c,b7a45ed0fd02..ea44fe7671fa --- a/src/cls/Makefile.am +++ b/src/cls/Makefile.am @@@ -40,9 -40,14 +40,14 @@@ radoslib_LTLIBRARIES += libcls_statelog libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' radoslib_LTLIBRARIES += libcls_replica_log.la + libcls_user_la_SOURCES = cls/user/cls_user.cc + libcls_user_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) + libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' + radoslib_LTLIBRARIES += libcls_user.la + libcls_rgw_la_SOURCES = \ cls/rgw/cls_rgw.cc \ cls/rgw/cls_rgw_ops.cc \ diff --cc src/common/config_opts.h index ad412e43c147,3a4ce7463260..98aa2262c56e --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@@ -756,10 -730,11 +756,16 @@@ OPTION(rgw_replica_log_obj_prefix, OPT_ OPTION(rgw_bucket_quota_ttl, OPT_INT, 600) // time for cached bucket stats to be cached within rgw instance OPTION(rgw_bucket_quota_soft_threshold, OPT_DOUBLE, 0.95) // threshold from which we don't rely on cached info for quota decisions OPTION(rgw_bucket_quota_cache_size, OPT_INT, 10000) // number of entries in bucket quota cache ++ +OPTION(rgw_expose_bucket, OPT_BOOL, false) // Return the bucket name in the 'Bucket' response header + +OPTION(rgw_frontends, OPT_STR, "") // alternative front ends + + OPTION(rgw_user_quota_bucket_sync_interval, OPT_INT, 180) // time period for accumulating modified buckets before syncing stats + OPTION(rgw_user_quota_sync_interval, OPT_INT, 3600 * 24) // time period for accumulating modified buckets before syncing entire user stats + OPTION(rgw_user_quota_sync_idle_users, OPT_BOOL, false) // whether stats for idle users be fully synced + OPTION(rgw_user_quota_sync_wait_time, OPT_INT, 3600 * 24) // min time between two full stats syc for non-idle users + OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter // This will be set to true when it is safe to start threads. diff --cc src/rgw/rgw_bucket.h index 7cd62365b997,b302238328b3..f113590f10e3 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@@ -102,10 -105,10 +105,10 @@@ extern int rgw_read_user_buckets(RGWRad extern int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true); extern int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name, bool update_entrypoint = true); - extern int rgw_remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object); - extern int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children); + extern int rgw_remove_object(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, std::string& object); + extern int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children); -extern int rgw_bucket_set_attrs(RGWRados *store, rgw_bucket& obj, +extern int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info, map& attrs, map* rmattrs, RGWObjVersionTracker *objv_tracker); diff --cc src/rgw/rgw_common.h index b41c6b3b91ed,cd10ceb05c5b..30f8e95b7a77 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@@ -30,8 -30,8 +30,9 @@@ #include "rgw_acl.h" #include "rgw_cors.h" #include "rgw_quota.h" +#include "rgw_string.h" #include "cls/version/cls_version_types.h" + #include "cls/user/cls_user_types.h" #include "include/rados/librados.hpp" using namespace std; @@@ -434,12 -432,12 +435,13 @@@ struct RGWUserInf string default_placement; list placement_tags; RGWQuotaInfo bucket_quota; + map temp_url_keys; + RGWQuotaInfo user_quota; RGWUserInfo() : auid(0), suspended(0), max_buckets(RGW_DEFAULT_MAX_BUCKETS), op_mask(RGW_OP_TYPE_ALL), system(0) {} void encode(bufferlist& bl) const { -- ENCODE_START(15, 9, bl); ++ ENCODE_START(16, 9, bl); ::encode(auid, bl); string access_key; string secret_key; @@@ -475,11 -473,11 +477,12 @@@ ::encode(default_placement, bl); ::encode(placement_tags, bl); ::encode(bucket_quota, bl); + ::encode(temp_url_keys, bl); + ::encode(user_quota, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { -- DECODE_START_LEGACY_COMPAT_LEN_32(13, 9, 9, bl); ++ DECODE_START_LEGACY_COMPAT_LEN_32(16, 9, 9, bl); if (struct_v >= 2) ::decode(auid, bl); else auid = CEPH_AUTH_UID_DEFAULT; string access_key; @@@ -536,8 -534,8 +539,11 @@@ ::decode(bucket_quota, bl); } if (struct_v >= 15) { + ::decode(temp_url_keys, bl); + } ++ if (struct_v >= 16) { + ::decode(user_quota, bl); + } DECODE_FINISH(bl); } void dump(Formatter *f) const; diff --cc src/rgw/rgw_json_enc.cc index 720b461a6872,0f03df3c49cb..231475b60e9e --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@@ -397,7 -397,7 +397,8 @@@ void RGWUserInfo::dump(Formatter *f) co encode_json("default_placement", default_placement, f); encode_json("placement_tags", placement_tags, f); encode_json("bucket_quota", bucket_quota, f); + encode_json("user_quota", user_quota, f); + encode_json("temp_url_keys", temp_url_keys, f); } @@@ -449,7 -449,7 +450,8 @@@ void RGWUserInfo::decode_json(JSONObj * JSONDecoder::decode_json("default_placement", default_placement, obj); JSONDecoder::decode_json("placement_tags", placement_tags, obj); JSONDecoder::decode_json("bucket_quota", bucket_quota, obj); + JSONDecoder::decode_json("user_quota", user_quota, obj); + JSONDecoder::decode_json("temp_url_keys", temp_url_keys, obj); } void RGWQuotaInfo::dump(Formatter *f) const diff --cc src/rgw/rgw_quota.cc index 89611f585876,6487b1f8badb..130776c3b8cd --- a/src/rgw/rgw_quota.cc +++ b/src/rgw/rgw_quota.cc @@@ -123,9 -230,34 +230,34 @@@ public }; - int AsyncRefreshHandler::init_fetch() + template + void RGWQuotaCache::adjust_stats(const string& user, rgw_bucket& bucket, int objs_delta, + uint64_t added_bytes, uint64_t removed_bytes) + { + RGWQuotaStatsUpdate update(objs_delta, added_bytes, removed_bytes); + map_find_and_update(user, bucket, &update); + + data_modified(user, bucket); + } + + class BucketAsyncRefreshHandler : public RGWQuotaCache::AsyncRefreshHandler, + public RGWGetBucketStats_CB { + string user; + public: + BucketAsyncRefreshHandler(RGWRados *_store, RGWQuotaCache *_cache, + const string& _user, rgw_bucket& _bucket) : + RGWQuotaCache::AsyncRefreshHandler(_store, _cache), + RGWGetBucketStats_CB(_bucket), user(_user) {} + + void drop_reference() { put(); } + void handle_response(int r); + int init_fetch(); + }; + + int BucketAsyncRefreshHandler::init_fetch() { ldout(store->ctx(), 20) << "initiating async quota refresh for bucket=" << bucket << dendl; - map bucket_stats; ++ int r = store->get_bucket_stats_async(bucket, this); if (r < 0) { ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl; diff --cc src/rgw/rgw_rados.cc index a2515b587b32,af3afe51a92c..8c8dd5f24802 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@@ -4819,42 -4857,7 +4893,42 @@@ int RGWRados::get_bucket_entrypoint_inf return 0; } +int RGWRados::convert_old_bucket_info(void *ctx, string& bucket_name) +{ + RGWBucketEntryPoint entry_point; + time_t ep_mtime; + RGWObjVersionTracker ot; + map attrs; + RGWBucketInfo info; + + ldout(cct, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket_name << dendl; + + int ret = get_bucket_entrypoint_info(ctx, bucket_name, entry_point, &ot, &ep_mtime, &attrs); + if (ret < 0) { + ldout(cct, 0) << "ERROR: get_bucket_entrypont_info() returned " << ret << " bucket=" << bucket_name << dendl; + return ret; + } + + if (!entry_point.has_bucket_info) { + /* already converted! */ + return 0; + } + + info = entry_point.old_bucket_info; + info.bucket.oid = bucket_name; + info.ep_objv = ot.read_version; + + ot.generate_new_write_ver(cct); + + ret = put_linked_bucket_info(info, false, ep_mtime, &ot.write_version, &attrs, true); + if (ret < 0) { + ldout(cct, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl; + } + + return 0; +} + - int RGWRados::get_bucket_info(void *ctx, string& bucket_name, RGWBucketInfo& info, + int RGWRados::get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info, time_t *pmtime, map *pattrs) { bufferlist bl; diff --cc src/rgw/rgw_rados.h index 67c540eb5d3a,fc40eb2e8092..64b9ef062ab6 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@@ -1337,8 -1363,7 +1365,8 @@@ public int get_bucket_instance_info(void *ctx, rgw_bucket& bucket, RGWBucketInfo& info, time_t *pmtime, map *pattrs); int get_bucket_instance_from_oid(void *ctx, string& oid, RGWBucketInfo& info, time_t *pmtime, map *pattrs); + int convert_old_bucket_info(void *ctx, string& bucket_name); - virtual int get_bucket_info(void *ctx, string& bucket_name, RGWBucketInfo& info, + virtual int get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info, time_t *pmtime, map *pattrs = NULL); virtual int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t mtime, obj_version *pep_objv, map *pattrs, bool create_entry_point); diff --cc src/rgw/rgw_user.cc index de220e005520,3a70ee24b374..f620f9b69dba --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@@ -1685,14 -1725,9 +1725,17 @@@ int RGWUser::execute_add(RGWUserAdminOp if (op_state.has_bucket_quota()) user_info.bucket_quota = op_state.get_bucket_quota(); + if (op_state.temp_url_key_specified) { + map::iterator iter; + for (iter = op_state.temp_url_keys.begin(); + iter != op_state.temp_url_keys.end(); ++iter) { + user_info.temp_url_keys[iter->first] = iter->second; + } + } + + if (op_state.has_user_quota()) + user_info.user_quota = op_state.get_user_quota(); + // update the request op_state.set_user_info(user_info); op_state.set_populated(); diff --cc src/rgw/rgw_user.h index 2f4b328ef0c7,be3ebcfdf02b..3cd08d30c77a --- a/src/rgw/rgw_user.h +++ b/src/rgw/rgw_user.h @@@ -424,7 -426,7 +433,8 @@@ struct RGWUserAdminOpState subuser_params_checked = false; user_params_checked = false; bucket_quota_specified = false; + temp_url_key_specified = false; + user_quota_specified = false; } };