From: Zhang Shaowen Date: Wed, 21 Jun 2017 12:05:16 +0000 (+0800) Subject: rgw: S3 lifecycle now supports expiration date X-Git-Tag: v12.1.1~4^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d02d0172865e2fb801a0cbcf07b12bd5597093c;p=ceph.git rgw: S3 lifecycle now supports expiration date Signed-off-by: Zhang Shaowen --- diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 848322d8957..0a33d4716b9 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -32,16 +32,17 @@ bool LCRule::validate() if (id.length() > MAX_ID_LEN) { return false; } - else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) { + else if(!expiration.has_days() && !expiration.has_date() && + !noncur_expiration.has_days() && !mp_expiration.has_days() && !dm_expiration) { return false; } - else if (!expiration.empty() && expiration.get_days() <= 0) { + else if (expiration.has_days() && expiration.get_days() <= 0) { return false; } - else if (!noncur_expiration.empty() && noncur_expiration.get_days() <=0) { + else if (noncur_expiration.has_days() && noncur_expiration.get_days() <=0) { return false; } - else if (!mp_expiration.empty() && mp_expiration.get_days() <= 0) { + else if (mp_expiration.has_days() && mp_expiration.get_days() <= 0) { return false; } return true; @@ -60,13 +61,16 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule) if (rule->get_status().compare("Enabled") == 0) { op.status = true; } - if (!rule->get_expiration().empty()) { + if (rule->get_expiration().has_days()) { op.expiration = rule->get_expiration().get_days(); } - if (!rule->get_noncur_expiration().empty()) { + if (rule->get_expiration().has_date()) { + op.expiration_date = ceph::from_iso_8601(rule->get_expiration().get_date()); + } + if (rule->get_noncur_expiration().has_days()) { op.noncur_expiration = rule->get_noncur_expiration().get_days(); } - if (!rule->get_mp_expiration().empty()) { + if (rule->get_mp_expiration().has_days()) { op.mp_expiration = rule->get_mp_expiration().get_days(); } op.dm_expiration = rule->get_dm_expiration(); @@ -92,6 +96,19 @@ int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule) return 0; } +bool RGWLifecycleConfiguration::has_same_action(const lc_op& first, const lc_op& second) { + if ((first.expiration > 0 || first.expiration_date != boost::none) && + (second.expiration > 0 || second.expiration_date != boost::none)) { + return true; + } else if (first.noncur_expiration > 0 && second.noncur_expiration > 0) { + return true; + } else if (first.mp_expiration > 0 && second.mp_expiration > 0) { + return true; + } else { + return false; + } +} + //Rules are conflicted: if one rule's prefix starts with other rule's prefix, and these two rules //define same action. bool RGWLifecycleConfiguration::validate() @@ -107,9 +124,7 @@ bool RGWLifecycleConfiguration::validate() string c_pre = cur_iter->first; string n_pre = next_iter->first; if (n_pre.compare(0, c_pre.length(), c_pre) == 0) { - if ((cur_iter->second.expiration > 0 && next_iter->second.expiration > 0) || - (cur_iter->second.noncur_expiration > 0 && next_iter->second.noncur_expiration > 0) || - (cur_iter->second.mp_expiration > 0 && next_iter->second.mp_expiration > 0)) { + if (has_same_action(cur_iter->second, next_iter->second)) { return false; } else { ++next_iter; @@ -346,7 +361,12 @@ int RGWLC::bucket_lc_process(string& shard_id) list_op.params.list_versions = bucket_info.versioned(); if (!bucket_info.versioned()) { for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) { - if (!prefix_iter->second.status || prefix_iter->second.expiration <=0) { + if (!prefix_iter->second.status || + (prefix_iter->second.expiration <=0 && prefix_iter->second.expiration_date == boost::none)) { + continue; + } + if (prefix_iter->second.expiration_date != boost::none && + ceph_clock_now() < ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date)) { continue; } list_op.params.prefix = prefix_iter->first; @@ -361,17 +381,22 @@ int RGWLC::bucket_lc_process(string& shard_id) ldout(cct, 0) << "ERROR: store->list_objects():" <key); if (!key.ns.empty()) { continue; } - - if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration)) { + if (prefix_iter->second.expiration_date != boost::none) { + //we have checked it before + is_expired = true; + } else { + is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration); + } + if (is_expired) { RGWObjectCtx rctx(store); rgw_obj obj(bucket_info.bucket, key); RGWObjState *state; @@ -396,6 +421,7 @@ int RGWLC::bucket_lc_process(string& shard_id) rgw_obj_key pre_marker; for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) { if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 + && prefix_iter->second.expiration_date == boost::none && prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) { continue; } @@ -427,10 +453,13 @@ int RGWLC::bucket_lc_process(string& shard_id) bool remove_indeed = true; int expiration; bool skip_expiration; + bool is_expired; for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) { skip_expiration = false; + is_expired = false; if (obj_iter->is_current()) { - if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) { + if (prefix_iter->second.expiration <= 0 && prefix_iter->second.expiration_date == boost::none + && !prefix_iter->second.dm_expiration) { continue; } if (obj_iter->is_delete_marker()) { @@ -450,8 +479,14 @@ int RGWLC::bucket_lc_process(string& shard_id) } mtime = obj_iter->meta.mtime; expiration = prefix_iter->second.expiration; - if (!skip_expiration && expiration <= 0) { + if (!skip_expiration && expiration <= 0 && prefix_iter->second.expiration_date == boost::none) { continue; + } else if (!skip_expiration) { + if (expiration > 0) { + is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration); + } else { + is_expired = now >= ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date); + } } } else { if (prefix_iter->second.noncur_expiration <=0) { @@ -460,8 +495,9 @@ int RGWLC::bucket_lc_process(string& shard_id) remove_indeed = true; mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime; expiration = prefix_iter->second.noncur_expiration; + is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration); } - if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) { + if (skip_expiration || is_expired) { if (obj_iter->is_visible()) { RGWObjectCtx rctx(store); rgw_obj obj(bucket_info.bucket, obj_iter->key); diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index 9609c952c45..d9df3d2a476 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -12,6 +12,7 @@ #include "include/rados/librados.hpp" #include "common/Mutex.h" #include "common/Cond.h" +#include "common/iso_8601.h" #include "common/Thread.h" #include "rgw_common.h" #include "rgw_rados.h" @@ -38,18 +39,24 @@ class LCExpiration { protected: string days; + //At present only current object has expiration date + string date; public: LCExpiration() {} ~LCExpiration() {} void encode(bufferlist& bl) const { - ENCODE_START(2, 2, bl); + ENCODE_START(3, 2, bl); ::encode(days, bl); + ::encode(date, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl); + DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); ::decode(days, bl); + if (struct_v >= 3) { + ::decode(date, bl); + } DECODE_FINISH(bl); } void dump(Formatter *f) const; @@ -59,8 +66,15 @@ public: return days; } int get_days() {return atoi(days.c_str()); } - bool empty() const{ - return days.empty(); + bool has_days() const { + return !days.empty(); + } + void set_date(const string& _date) { date = _date; } + string get_date() const{ + return date; + } + bool has_date() const { + return !date.empty(); } }; WRITE_CLASS_ENCODER(LCExpiration) @@ -179,6 +193,7 @@ struct lc_op int expiration; int noncur_expiration; int mp_expiration; + boost::optional expiration_date; lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {} @@ -191,6 +206,7 @@ protected: map prefix_map; multimap rule_map; bool _add_rule(LCRule *rule); + bool has_same_action(const lc_op& first, const lc_op& second); public: RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {} RGWLifecycleConfiguration() : cct(NULL) {} diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc index eb58a0c9f41..525cd0a4353 100644 --- a/src/rgw/rgw_lc_s3.cc +++ b/src/rgw/rgw_lc_s3.cc @@ -16,8 +16,10 @@ using namespace std; bool LCExpiration_S3::xml_end(const char * el) { LCDays_S3 *lc_days = static_cast(find_first("Days")); LCDeleteMarker_S3 *lc_dm = static_cast(find_first("ExpiredObjectDeleteMarker")); + LCDate_S3 *lc_date = static_cast(find_first("Date")); - if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) { + if ((!lc_days && !lc_dm && !lc_date) || (lc_days && lc_dm) + || (lc_days && lc_date) || (lc_dm && lc_date)) { return false; } if (lc_days) { @@ -27,6 +29,11 @@ bool LCExpiration_S3::xml_end(const char * el) { if (!dm_expiration) { return false; } + } else { + date = lc_date->get_data(); + if (boost::none == ceph::from_iso_8601(date)) { + return false; + } } return true; } @@ -96,8 +103,10 @@ bool LCRule_S3::xml_end(const char *el) { return false; } else { if (lc_expiration) { - if (!lc_expiration->empty()) { + if (lc_expiration->has_days()) { expiration.set_days(lc_expiration->get_days_str()); + } else if (lc_expiration->has_date()) { + expiration.set_date(lc_expiration->get_date()); } else { dm_expiration = lc_expiration->get_dm_expiration(); } @@ -118,15 +127,15 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) { out << "" << id << ""; out << "" << prefix << ""; out << "" << status << ""; - if (!expiration.empty() || dm_expiration) { - LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration); + if (expiration.has_days() || expiration.has_date() || dm_expiration) { + LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration); expir.to_xml(out); } - if (!noncur_expiration.empty()) { + if (noncur_expiration.has_days()) { LCNoncurExpiration_S3& noncur_expir = static_cast(noncur_expiration); noncur_expir.to_xml(out); } - if (!mp_expiration.empty()) { + if (mp_expiration.has_days()) { LCMPExpiration_S3& mp_expir = static_cast(mp_expiration); mp_expir.to_xml(out); } @@ -178,6 +187,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el) obj = new LCExpiration_S3(); } else if (strcmp(el, "Days") == 0) { obj = new LCDays_S3(); + } else if (strcmp(el, "Date") == 0) { + obj = new LCDate_S3(); } else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) { obj = new LCDeleteMarker_S3(); } else if (strcmp(el, "NoncurrentVersionExpiration") == 0) { diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h index 53c5f7e8ba5..8d74bcaae79 100644 --- a/src/rgw/rgw_lc_s3.h +++ b/src/rgw/rgw_lc_s3.h @@ -42,6 +42,14 @@ public: string& to_str() { return data; } }; +class LCDate_S3 : public XMLObj +{ +public: + LCDate_S3() {} + ~LCDate_S3() override {} + string& to_str() { return data; } +}; + class LCDeleteMarker_S3 : public XMLObj { public: @@ -56,26 +64,33 @@ private: bool dm_expiration; public: LCExpiration_S3(): dm_expiration(false) {} - LCExpiration_S3(string _days, bool _dm_expiration) { + LCExpiration_S3(string _days, string _date, bool _dm_expiration) { days = _days; + date = _date; dm_expiration = _dm_expiration; } ~LCExpiration_S3() override {} bool xml_end(const char *el) override; void to_xml(ostream& out) { + out << ""; if (dm_expiration) { - out << "" << "" << "true" << "" << ""; + out << "" << "true" << ""; + } else if (!days.empty()){ + out << "" << days << ""; } else { - out << "" << "" << days << ""<< ""; + out << "" << date << ""; } + out << ""; } void dump_xml(Formatter *f) const { f->open_object_section("Expiration"); if (dm_expiration) { encode_xml("ExpiredObjectDeleteMarker", "true", f); - } else { + } else if (!days.empty()) { encode_xml("Days", days, f); + } else { + encode_xml("Date", date, f); } f->close_section(); // Expiration } @@ -137,15 +152,15 @@ public: encode_xml("ID", id, f); encode_xml("Prefix", prefix, f); encode_xml("Status", status, f); - if (!expiration.empty() || dm_expiration) { - LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration); + if (expiration.has_days() || expiration.has_date() || dm_expiration) { + LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration); expir.dump_xml(f); } - if (!noncur_expiration.empty()) { + if (noncur_expiration.has_days()) { const LCNoncurExpiration_S3& noncur_expir = static_cast(noncur_expiration); noncur_expir.dump_xml(f); } - if (!mp_expiration.empty()) { + if (mp_expiration.has_days()) { const LCMPExpiration_S3& mp_expir = static_cast(mp_expiration); mp_expir.dump_xml(f); }