From 77853a41f5e70d016abb8619327f9c5174a068d6 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Thu, 14 Jul 2016 12:40:29 -0400 Subject: [PATCH] Lifecycle testing Add new config option - rgw_lc_debug_interval - that modifies the lifecycle algorithm. It treats days as being the configured number of seconds. This allows lifecycle to be tested in minutes, rather than days. Signed-off-by: Daniel Gryniewicz --- src/common/config_opts.h | 1 + src/rgw/rgw_lc.cc | 66 ++++++++++++++++++++++++++++------------ src/rgw/rgw_lc.h | 6 ++-- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 470e1d4083c71..e3b7500850d23 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1318,6 +1318,7 @@ OPTION(rgw_lifecycle_thread, OPT_INT, 1) //start lifecycle thread number per rad OPTION(rgw_lifecycle_work_time, OPT_STR, "00:00-06:00") //job process lc at 00:00-06:00s OPTION(rgw_lc_lock_max_time, OPT_INT, 60) // total run time for a single gc processor work OPTION(rgw_lc_max_objs, OPT_INT, 32) +OPTION(rgw_lc_debug_interval, OPT_INT, -1) // Debug run interval, in seconds OPTION(rgw_script_uri, OPT_STR, "") // alternative value for SCRIPT_URI if not set in request OPTION(rgw_request_uri, OPT_STR, "") // alternative value for REQUEST_URI if not set in request OPTION(rgw_swift_url, OPT_STR, "") // the swift url, being published by the internal swift auth diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 39b25dbbc08e4..8c5ec60a143de 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -58,11 +58,11 @@ void *RGWLC::LCWorker::entry() { break; utime_t end = ceph_clock_now(cct); - int secs = shedule_next_start_time(end); + int secs = schedule_next_start_time(start, end); time_t next_time = end + secs; char buf[30]; char *nt = ctime_r(&next_time, buf); - dout(5) << "shedule life cycle next start time: " << nt <_conf->rgw_lc_debug_interval > 0) { + /* We're debugging, so say we can run */ + return false; + } + bdt.tm_hour = 0; bdt.tm_min = 0; bdt.tm_sec = 0; @@ -156,18 +162,32 @@ int RGWLC::bucket_lc_prepare(int index) return 0; } +bool RGWLC::obj_has_expired(double timediff, int days) +{ + double cmp; + + if (cct->_conf->rgw_lc_debug_interval <= 0) { + /* Normal case, run properly */ + cmp = days*24*60*60; + } else { + /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */ + cmp = days*cct->_conf->rgw_lc_debug_interval; + } + + return (timediff >= cmp); +} + int RGWLC::bucket_lc_process(string& shard_id) { RGWLifecycleConfiguration config(cct); RGWBucketInfo bucket_info; map bucket_attrs; - string prefix, delimiter, marker, next_marker, no_ns, end_marker, list_versions; + string next_marker, no_ns, list_versions; bool is_truncated; bool default_config = false; int default_days = 0; vector objs; RGWObjectCtx obj_ctx(store); - map common_prefixes; vector result; result = split(shard_id, ':'); string bucket_tenant = result[0]; @@ -188,12 +208,6 @@ int RGWLC::bucket_lc_process(string& shard_id) RGWRados::Bucket target(store, bucket_info); RGWRados::Bucket::List list_op(&target); - list_op.params.prefix = prefix; - list_op.params.delim = delimiter; - list_op.params.marker = marker; - list_op.params.end_marker = end_marker; - list_op.params.list_versions = false; - map::iterator aiter = bucket_attrs.find(RGW_ATTR_LC); if (aiter == bucket_attrs.end()) return 0; @@ -211,7 +225,7 @@ int RGWLC::bucket_lc_process(string& shard_id) if (prefix_iter->first.empty()) { default_config = true; default_days = prefix_iter->second; - continue; + break; } } @@ -220,7 +234,7 @@ int RGWLC::bucket_lc_process(string& shard_id) objs.clear(); list_op.params.marker = list_op.get_next_marker(); - ret = list_op.list_objects(1000, &objs, &common_prefixes, &is_truncated); + ret = list_op.list_objects(1000, &objs, NULL, &is_truncated); if (ret < 0) { if (ret == -ENOENT) return 0; @@ -256,7 +270,7 @@ int RGWLC::bucket_lc_process(string& shard_id) } else { continue; } - if (now - ceph::real_clock::to_time_t((*obj_iter).mtime) >= days*24*60*60) { + if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) { RGWObjectCtx rctx(store); rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name); RGWObjState *state; @@ -286,7 +300,7 @@ int RGWLC::bucket_lc_process(string& shard_id) objs.clear(); list_op.params.marker = list_op.get_next_marker(); - ret = list_op.list_objects(1000, &objs, &common_prefixes, &is_truncated); + ret = list_op.list_objects(1000, &objs, NULL, &is_truncated); if (ret < 0) { if (ret == (-ENOENT)) @@ -300,7 +314,7 @@ int RGWLC::bucket_lc_process(string& shard_id) utime_t now = ceph_clock_now(cct); for (obj_iter = objs.begin(); obj_iter != objs.end(); obj_iter++) { - if (now - ceph::real_clock::to_time_t((*obj_iter).mtime) >= days*24*60*60) { + if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) { RGWObjectCtx rctx(store); rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name); RGWObjState *state; @@ -511,17 +525,28 @@ bool RGWLC::LCWorker::should_work(utime_t& now) struct tm bdt; time_t tt = now.sec(); localtime_r(&tt, &bdt); - if ((bdt.tm_hour*60 + bdt.tm_min >= start_hour*60 + start_minute)|| - (bdt.tm_hour*60 + bdt.tm_min <= end_hour*60 + end_minute)) { - return true; + + if (cct->_conf->rgw_lc_debug_interval > 0) { + /* We're debugging, so say we can run */ + return true; + } else if ((bdt.tm_hour*60 + bdt.tm_min >= start_hour*60 + start_minute) || + (bdt.tm_hour*60 + bdt.tm_min <= end_hour*60 + end_minute)) { + return true; } else { - return false; + return false; } } -int RGWLC::LCWorker::shedule_next_start_time(utime_t& now) +int RGWLC::LCWorker::schedule_next_start_time(utime_t &start, utime_t& now) { + if (cct->_conf->rgw_lc_debug_interval > 0) { + int secs = start + cct->_conf->rgw_lc_debug_interval - now; + if (secs < 0) + secs = 0; + return (secs); + } + int start_hour; int start_minute; int end_hour; @@ -536,6 +561,7 @@ int RGWLC::LCWorker::shedule_next_start_time(utime_t& now) bdt.tm_min = start_minute; bdt.tm_sec = 0; nt = mktime(&bdt); + return (nt+24*60*60 - tt); } diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index dbe9cabbfc3c1..d34f63dd38488 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -193,12 +193,11 @@ class RGWLC { void *entry(); void stop(); bool should_work(utime_t& now); - int shedule_next_start_time(utime_t& now); + int schedule_next_start_time(utime_t& start, utime_t& now); }; public: LCWorker *worker; -public: RGWLC() : cct(NULL), store(NULL), worker(NULL) {} ~RGWLC() { stop_processor(); @@ -219,6 +218,9 @@ public: bool going_down(); void start_processor(); void stop_processor(); + + private: + bool obj_has_expired(double timediff, int days); }; -- 2.39.5