From: Zhang Shaowen Date: Wed, 19 Apr 2017 07:39:40 +0000 (+0800) Subject: rgw: add support for delete marker expiration in s3 lifecycle. X-Git-Tag: v12.1.0~46^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F14703%2Fhead;p=ceph.git rgw: add support for delete marker expiration in s3 lifecycle. Fixes: http://tracker.ceph.com/issues/19730 Signed-off-by: Zhang Shaowen --- diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 1b7ce3c572e22..e45326c6fe900 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -32,7 +32,7 @@ bool LCRule::validate() if (id.length() > MAX_ID_LEN) { return false; } - else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty()) { + else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) { return false; } else if (!expiration.empty() && expiration.get_days() <= 0) { @@ -69,6 +69,7 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule) if (!rule->get_mp_expiration().empty()) { op.mp_expiration = rule->get_mp_expiration().get_days(); } + op.dm_expiration = rule->get_dm_expiration(); auto ret = prefix_map.insert(pair(rule->get_prefix(), op)); return ret.second; } @@ -228,17 +229,16 @@ int RGWLC::bucket_lc_prepare(int index) bool RGWLC::obj_has_expired(double timediff, int days) { - double cmp; - - if (cct->_conf->rgw_lc_debug_interval <= 0) { - /* Normal case, run properly */ - cmp = days*24*60*60; - } else { - /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */ - cmp = days*cct->_conf->rgw_lc_debug_interval; - } + double cmp; + if (cct->_conf->rgw_lc_debug_interval <= 0) { + /* Normal case, run properly */ + cmp = days*24*60*60; + } else { + /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */ + cmp = days*cct->_conf->rgw_lc_debug_interval; + } - return (timediff >= cmp); + return (timediff >= cmp); } int RGWLC::remove_expired_obj(RGWBucketInfo& bucket_info, rgw_obj_key obj_key, bool remove_indeed) @@ -395,7 +395,8 @@ int RGWLC::bucket_lc_process(string& shard_id) //bucket versioning is enabled or suspended rgw_obj_key pre_marker; for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) { - if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 && prefix_iter->second.noncur_expiration <= 0)) { + if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 + && prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) { continue; } if (prefix_iter != prefix_map.begin() && @@ -425,9 +426,11 @@ int RGWLC::bucket_lc_process(string& shard_id) ceph::real_time mtime; bool remove_indeed = true; int expiration; + bool skip_expiration; for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) { + skip_expiration = false; if (obj_iter->is_current()) { - if (prefix_iter->second.expiration <= 0) { + if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) { continue; } if (obj_iter->is_delete_marker()) { @@ -440,12 +443,16 @@ int RGWLC::bucket_lc_process(string& shard_id) } else if (obj_iter->key.name.compare((obj_iter + 1)->key.name) == 0) { //*obj_iter is delete marker and isn't the only version, do nothing. continue; } + skip_expiration = prefix_iter->second.dm_expiration; remove_indeed = true; //we should remove the delete marker if it's the only version } else { remove_indeed = false; } mtime = obj_iter->meta.mtime; expiration = prefix_iter->second.expiration; + if (!skip_expiration && expiration <= 0) { + continue; + } } else { if (prefix_iter->second.noncur_expiration <=0) { continue; @@ -454,7 +461,7 @@ int RGWLC::bucket_lc_process(string& shard_id) mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime; expiration = prefix_iter->second.noncur_expiration; } - if (obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) { + if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) { if (obj_iter->is_visible()) { RGWObjectCtx rctx(store); rgw_obj obj(bucket_info.bucket, obj_iter->key); diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index 480ba3fe513bb..9609c952c45e9 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -55,6 +55,9 @@ public: void dump(Formatter *f) const; // static void generate_test_instances(list& o); void set_days(const string& _days) { days = _days; } + string get_days_str() const{ + return days; + } int get_days() {return atoi(days.c_str()); } bool empty() const{ return days.empty(); @@ -71,6 +74,7 @@ protected: LCExpiration expiration; LCExpiration noncur_expiration; LCExpiration mp_expiration; + bool dm_expiration; public: @@ -102,6 +106,10 @@ public: return mp_expiration; } + bool get_dm_expiration() { + return dm_expiration; + } + void set_id(string*_id) { id = *_id; } @@ -126,20 +134,25 @@ public: mp_expiration = *_mp_expiration; } + void set_dm_expiration(bool _dm_expiration) { + dm_expiration = _dm_expiration; + } + bool validate(); void encode(bufferlist& bl) const { - ENCODE_START(3, 1, bl); + ENCODE_START(4, 1, bl); ::encode(id, bl); ::encode(prefix, bl); ::encode(status, bl); ::encode(expiration, bl); ::encode(noncur_expiration, bl); ::encode(mp_expiration, bl); + ::encode(dm_expiration, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN(3, 1, 1, bl); + DECODE_START_LEGACY_COMPAT_LEN(4, 1, 1, bl); ::decode(id, bl); ::decode(prefix, bl); ::decode(status, bl); @@ -150,6 +163,9 @@ public: if (struct_v >= 3) { ::decode(mp_expiration, bl); } + if (struct_v >= 4) { + ::decode(dm_expiration, bl); + } DECODE_FINISH(bl); } @@ -159,11 +175,12 @@ WRITE_CLASS_ENCODER(LCRule) struct lc_op { bool status; + bool dm_expiration; int expiration; int noncur_expiration; int mp_expiration; - lc_op() : status(false), expiration(0), noncur_expiration(0), mp_expiration(0) {} + lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {} }; diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc index 99ed7b70cd688..eb58a0c9f4108 100644 --- a/src/rgw/rgw_lc_s3.cc +++ b/src/rgw/rgw_lc_s3.cc @@ -15,10 +15,19 @@ using namespace std; bool LCExpiration_S3::xml_end(const char * el) { LCDays_S3 *lc_days = static_cast(find_first("Days")); + LCDeleteMarker_S3 *lc_dm = static_cast(find_first("ExpiredObjectDeleteMarker")); - if (!lc_days) + if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) { return false; - days = lc_days->get_data(); + } + if (lc_days) { + days = lc_days->get_data(); + } else if (lc_dm) { + dm_expiration = lc_dm->get_data().compare("true") == 0; + if (!dm_expiration) { + return false; + } + } return true; } @@ -61,6 +70,7 @@ bool LCRule_S3::xml_end(const char *el) { id.clear(); prefix.clear(); status.clear(); + dm_expiration = false; lc_id = static_cast(find_first("ID")); if (!lc_id) @@ -86,7 +96,11 @@ bool LCRule_S3::xml_end(const char *el) { return false; } else { if (lc_expiration) { - expiration = *lc_expiration; + if (!lc_expiration->empty()) { + expiration.set_days(lc_expiration->get_days_str()); + } else { + dm_expiration = lc_expiration->get_dm_expiration(); + } } if (lc_noncur_expiration) { noncur_expiration = *lc_noncur_expiration; @@ -104,8 +118,8 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) { out << "" << id << ""; out << "" << prefix << ""; out << "" << status << ""; - if (!expiration.empty()) { - LCExpiration_S3& expir = static_cast(expiration); + if (!expiration.empty() || dm_expiration) { + LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration); expir.to_xml(out); } if (!noncur_expiration.empty()) { @@ -164,6 +178,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el) obj = new LCExpiration_S3(); } else if (strcmp(el, "Days") == 0) { obj = new LCDays_S3(); + } else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) { + obj = new LCDeleteMarker_S3(); } else if (strcmp(el, "NoncurrentVersionExpiration") == 0) { obj = new LCNoncurExpiration_S3(); } else if (strcmp(el, "NoncurrentDays") == 0) { diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h index 52f6987e48775..53c5f7e8ba59e 100644 --- a/src/rgw/rgw_lc_s3.h +++ b/src/rgw/rgw_lc_s3.h @@ -42,20 +42,50 @@ public: string& to_str() { return data; } }; +class LCDeleteMarker_S3 : public XMLObj +{ +public: + LCDeleteMarker_S3() {} + ~LCDeleteMarker_S3() override {} + string& to_str() { return data; } +}; + class LCExpiration_S3 : public LCExpiration, public XMLObj { +private: + bool dm_expiration; public: - LCExpiration_S3() {} + LCExpiration_S3(): dm_expiration(false) {} + LCExpiration_S3(string _days, bool _dm_expiration) { + days = _days; + dm_expiration = _dm_expiration; + } ~LCExpiration_S3() override {} bool xml_end(const char *el) override; void to_xml(ostream& out) { - out << "" << "" << days << ""<< ""; + if (dm_expiration) { + out << "" << "" << "true" << "" << ""; + } else { + out << "" << "" << days << ""<< ""; + } } void dump_xml(Formatter *f) const { - f->open_object_section("Expiration"); - encode_xml("Days", days, f); - f->close_section(); // Expiration + f->open_object_section("Expiration"); + if (dm_expiration) { + encode_xml("ExpiredObjectDeleteMarker", "true", f); + } else { + encode_xml("Days", days, f); + } + f->close_section(); // Expiration + } + + void set_dm_expiration(bool _dm_expiration) { + dm_expiration = _dm_expiration; + } + + bool get_dm_expiration() { + return dm_expiration; } }; @@ -107,8 +137,8 @@ public: encode_xml("ID", id, f); encode_xml("Prefix", prefix, f); encode_xml("Status", status, f); - if (!expiration.empty()) { - const LCExpiration_S3& expir = static_cast(expiration); + if (!expiration.empty() || dm_expiration) { + LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration); expir.dump_xml(f); } if (!noncur_expiration.empty()) {