rgw: S3 lifecycle now supports expiration date

Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 848322d89574b54eddec641446c9f20d5f3c1503..0a33d4716b97600a273acef79ece88500578ac71 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -32,16 +32,17 @@ bool LCRule::validate()
   if (id.length() > MAX_ID_LEN) {
     return false;
   }
-  else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) {
+  else if(!expiration.has_days() && !expiration.has_date() && 
+    !noncur_expiration.has_days() && !mp_expiration.has_days() && !dm_expiration) {
     return false;
   }
-  else if (!expiration.empty() && expiration.get_days() <= 0) {
+  else if (expiration.has_days() && expiration.get_days() <= 0) {
     return false;
   }
-  else if (!noncur_expiration.empty() && noncur_expiration.get_days() <=0) {
+  else if (noncur_expiration.has_days() && noncur_expiration.get_days() <=0) {
     return false;
   }
-  else if (!mp_expiration.empty() && mp_expiration.get_days() <= 0) {
+  else if (mp_expiration.has_days() && mp_expiration.get_days() <= 0) {
     return false;
   }
   return true;
@@ -60,13 +61,16 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
   if (rule->get_status().compare("Enabled") == 0) {
     op.status = true;
   }
-  if (!rule->get_expiration().empty()) {
+  if (rule->get_expiration().has_days()) {
     op.expiration = rule->get_expiration().get_days();
   }
-  if (!rule->get_noncur_expiration().empty()) {
+  if (rule->get_expiration().has_date()) {
+    op.expiration_date = ceph::from_iso_8601(rule->get_expiration().get_date());
+  }
+  if (rule->get_noncur_expiration().has_days()) {
     op.noncur_expiration = rule->get_noncur_expiration().get_days();
   }
-  if (!rule->get_mp_expiration().empty()) {
+  if (rule->get_mp_expiration().has_days()) {
     op.mp_expiration = rule->get_mp_expiration().get_days();
   }
   op.dm_expiration = rule->get_dm_expiration();
@@ -92,6 +96,19 @@ int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
   return 0;
 }
 
+bool RGWLifecycleConfiguration::has_same_action(const lc_op& first, const lc_op& second) {
+  if ((first.expiration > 0 || first.expiration_date != boost::none) && 
+    (second.expiration > 0 || second.expiration_date != boost::none)) {
+    return true;
+  } else if (first.noncur_expiration > 0 && second.noncur_expiration > 0) {
+    return true;
+  } else if (first.mp_expiration > 0 && second.mp_expiration > 0) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
 //Rules are conflicted: if one rule's prefix starts with other rule's prefix, and these two rules
 //define same action. 
 bool RGWLifecycleConfiguration::validate() 
@@ -107,9 +124,7 @@ bool RGWLifecycleConfiguration::validate()
       string c_pre = cur_iter->first;
       string n_pre = next_iter->first;
       if (n_pre.compare(0, c_pre.length(), c_pre) == 0) {
-        if ((cur_iter->second.expiration > 0 && next_iter->second.expiration > 0) ||
-          (cur_iter->second.noncur_expiration > 0 && next_iter->second.noncur_expiration > 0) || 
-          (cur_iter->second.mp_expiration > 0 && next_iter->second.mp_expiration > 0)) {
+        if (has_same_action(cur_iter->second, next_iter->second)) {
           return false;
         } else {
           ++next_iter;
@@ -346,7 +361,12 @@ int RGWLC::bucket_lc_process(string& shard_id)
   list_op.params.list_versions = bucket_info.versioned();
   if (!bucket_info.versioned()) {
     for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
-      if (!prefix_iter->second.status || prefix_iter->second.expiration <=0) {
+      if (!prefix_iter->second.status || 
+        (prefix_iter->second.expiration <=0 && prefix_iter->second.expiration_date == boost::none)) {
+        continue;
+      }
+      if (prefix_iter->second.expiration_date != boost::none && 
+        ceph_clock_now() < ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date)) {
         continue;
       }
       list_op.params.prefix = prefix_iter->first;
@@ -361,17 +381,22 @@ int RGWLC::bucket_lc_process(string& shard_id)
           ldout(cct, 0) << "ERROR: store->list_objects():" <<dendl;
           return ret;
         }
-
+        
         utime_t now = ceph_clock_now();
-
+        bool is_expired;
         for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
           rgw_obj_key key(obj_iter->key);
 
           if (!key.ns.empty()) {
             continue;
           }
-
-          if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration)) {
+          if (prefix_iter->second.expiration_date != boost::none) {
+            //we have checked it before
+            is_expired = true;
+          } else {
+            is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration);
+          }
+          if (is_expired) {
             RGWObjectCtx rctx(store);
             rgw_obj obj(bucket_info.bucket, key);
             RGWObjState *state;
@@ -396,6 +421,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
     rgw_obj_key pre_marker;
     for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
       if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 
+        && prefix_iter->second.expiration_date == boost::none
         && prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) {
         continue;
       }
@@ -427,10 +453,13 @@ int RGWLC::bucket_lc_process(string& shard_id)
         bool remove_indeed = true;
         int expiration;
         bool skip_expiration;
+        bool is_expired;
         for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
           skip_expiration = false;
+          is_expired = false;
           if (obj_iter->is_current()) {
-            if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) {
+            if (prefix_iter->second.expiration <= 0 && prefix_iter->second.expiration_date == boost::none
+              && !prefix_iter->second.dm_expiration) {
               continue;
             }
             if (obj_iter->is_delete_marker()) {
@@ -450,8 +479,14 @@ int RGWLC::bucket_lc_process(string& shard_id)
             }
             mtime = obj_iter->meta.mtime;
             expiration = prefix_iter->second.expiration;
-            if (!skip_expiration && expiration <= 0) {
+            if (!skip_expiration && expiration <= 0 && prefix_iter->second.expiration_date == boost::none) {
               continue;
+            } else if (!skip_expiration) {
+              if (expiration > 0) {
+                is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
+              } else {
+                is_expired = now >= ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date);
+              }
             }
           } else {
             if (prefix_iter->second.noncur_expiration <=0) {
@@ -460,8 +495,9 @@ int RGWLC::bucket_lc_process(string& shard_id)
             remove_indeed = true;
             mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime;
             expiration = prefix_iter->second.noncur_expiration;
+            is_expired = obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration);
           }
-          if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
+          if (skip_expiration || is_expired) {
             if (obj_iter->is_visible()) {
               RGWObjectCtx rctx(store);
               rgw_obj obj(bucket_info.bucket, obj_iter->key);

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index 9609c952c45e93a07bb23a5b291c4bdc0ef77ac9..d9df3d2a476eed530efba9feca9fda79df2f5b8a 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -12,6 +12,7 @@
 #include "include/rados/librados.hpp"
 #include "common/Mutex.h"
 #include "common/Cond.h"
+#include "common/iso_8601.h"
 #include "common/Thread.h"
 #include "rgw_common.h"
 #include "rgw_rados.h"
@@ -38,18 +39,24 @@ class LCExpiration
 {
 protected:
   string days;
+  //At present only current object has expiration date
+  string date;
 public:
   LCExpiration() {}
   ~LCExpiration() {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(2, 2, bl);
+    ENCODE_START(3, 2, bl);
     ::encode(days, bl);
+    ::encode(date, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
     ::decode(days, bl);
+    if (struct_v >= 3) {
+      ::decode(date, bl);
+    }
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
@@ -59,8 +66,15 @@ public:
     return days;
   }
   int get_days() {return atoi(days.c_str()); }
-  bool empty() const{
-    return days.empty();
+  bool has_days() const {
+    return !days.empty();
+  }
+  void set_date(const string& _date) { date = _date; }
+  string get_date() const{
+    return date;
+  }
+  bool has_date() const {
+    return !date.empty();
   }
 };
 WRITE_CLASS_ENCODER(LCExpiration)
@@ -179,6 +193,7 @@ struct lc_op
   int expiration;
   int noncur_expiration;
   int mp_expiration;
+  boost::optional<ceph::real_time> expiration_date;
 
   lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {}
   
@@ -191,6 +206,7 @@ protected:
   map<string, lc_op> prefix_map;
   multimap<string, LCRule> rule_map;
   bool _add_rule(LCRule *rule);
+  bool has_same_action(const lc_op& first, const lc_op& second);
 public:
   RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
   RGWLifecycleConfiguration() : cct(NULL) {}

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index eb58a0c9f4108c5c76e59f76a901d681f0d8b66f..525cd0a435320a7ac5b10a14c075e5160fd6e50c 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -16,8 +16,10 @@ using namespace std;
 bool LCExpiration_S3::xml_end(const char * el) {
   LCDays_S3 *lc_days = static_cast<LCDays_S3 *>(find_first("Days"));
   LCDeleteMarker_S3 *lc_dm = static_cast<LCDeleteMarker_S3 *>(find_first("ExpiredObjectDeleteMarker"));
+  LCDate_S3 *lc_date = static_cast<LCDate_S3 *>(find_first("Date"));
 
-  if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) {
+  if ((!lc_days && !lc_dm && !lc_date) || (lc_days && lc_dm) 
+      || (lc_days && lc_date) || (lc_dm && lc_date)) {
     return false;
   }
   if (lc_days) {
@@ -27,6 +29,11 @@ bool LCExpiration_S3::xml_end(const char * el) {
     if (!dm_expiration) {
       return false;
     }
+  } else {
+    date = lc_date->get_data();
+    if (boost::none == ceph::from_iso_8601(date)) {
+      return false;
+    }
   }
   return true;
 }
@@ -96,8 +103,10 @@ bool LCRule_S3::xml_end(const char *el) {
     return false;
   } else {
     if (lc_expiration) {
-      if (!lc_expiration->empty()) {
+      if (lc_expiration->has_days()) {
         expiration.set_days(lc_expiration->get_days_str());
+      } else if (lc_expiration->has_date()) {
+        expiration.set_date(lc_expiration->get_date());
       } else {
         dm_expiration = lc_expiration->get_dm_expiration();
       }
@@ -118,15 +127,15 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) {
   out << "<ID>" << id << "</ID>";
   out << "<Prefix>" << prefix << "</Prefix>";
   out << "<Status>" << status << "</Status>";
-  if (!expiration.empty() || dm_expiration) {
-    LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
+  if (expiration.has_days() || expiration.has_date() || dm_expiration) {
+    LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
     expir.to_xml(out);
   }
-  if (!noncur_expiration.empty()) {
+  if (noncur_expiration.has_days()) {
     LCNoncurExpiration_S3& noncur_expir = static_cast<LCNoncurExpiration_S3&>(noncur_expiration);
     noncur_expir.to_xml(out);
   }
-  if (!mp_expiration.empty()) {
+  if (mp_expiration.has_days()) {
     LCMPExpiration_S3& mp_expir = static_cast<LCMPExpiration_S3&>(mp_expiration);
     mp_expir.to_xml(out);
   }
@@ -178,6 +187,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el)
     obj = new LCExpiration_S3();
   } else if (strcmp(el, "Days") == 0) {
     obj = new LCDays_S3();
+  } else if (strcmp(el, "Date") == 0) {
+    obj = new LCDate_S3();
   } else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) {
     obj = new LCDeleteMarker_S3();
   } else if (strcmp(el, "NoncurrentVersionExpiration") == 0) {

diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h
index 53c5f7e8ba59eeaefc83d17f336a1f35c2a863c3..8d74bcaae791ea74f2ee838f5e9848b475483d78 100644 (file)
--- a/src/rgw/rgw_lc_s3.h
+++ b/src/rgw/rgw_lc_s3.h
@@ -42,6 +42,14 @@ public:
   string& to_str() { return data; }
 };
 
+class LCDate_S3 : public XMLObj
+{
+public:
+  LCDate_S3() {}
+  ~LCDate_S3() override {}
+  string& to_str() { return data; }
+};
+
 class LCDeleteMarker_S3 : public XMLObj
 {
 public:
@@ -56,26 +64,33 @@ private:
   bool dm_expiration;
 public:
   LCExpiration_S3(): dm_expiration(false) {}
-  LCExpiration_S3(string _days, bool _dm_expiration) {
+  LCExpiration_S3(string _days, string _date, bool _dm_expiration) {
     days = _days;
+    date = _date;
     dm_expiration = _dm_expiration;
   }
   ~LCExpiration_S3() override {}
 
   bool xml_end(const char *el) override;
   void to_xml(ostream& out) {
+    out << "<Expiration>";
     if (dm_expiration) {
-      out << "<Expiration>" << "<ExpiredObjectDeleteMarker>" << "true" << "</ExpiredObjectDeleteMarker>" << "</Expiration>";
+      out << "<ExpiredObjectDeleteMarker>" << "true" << "</ExpiredObjectDeleteMarker>";
+    } else if (!days.empty()){
+      out << "<Days>" << days << "</Days>";
     } else {
-      out << "<Expiration>" << "<Days>" << days << "</Days>"<< "</Expiration>";
+      out << "<Date>" << date << "</Date>";
     }
+    out << "</Expiration>";
   }
   void dump_xml(Formatter *f) const {
     f->open_object_section("Expiration");
     if (dm_expiration) {
       encode_xml("ExpiredObjectDeleteMarker", "true", f);
-    } else {
+    } else if (!days.empty()) {
       encode_xml("Days", days, f);
+    } else {
+      encode_xml("Date", date, f);
     }
     f->close_section(); // Expiration
   }
@@ -137,15 +152,15 @@ public:
     encode_xml("ID", id, f);
     encode_xml("Prefix", prefix, f);
     encode_xml("Status", status, f);
-    if (!expiration.empty() || dm_expiration) {
-      LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
+    if (expiration.has_days() || expiration.has_date() || dm_expiration) {
+      LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
       expir.dump_xml(f);
     }
-    if (!noncur_expiration.empty()) {
+    if (noncur_expiration.has_days()) {
       const LCNoncurExpiration_S3& noncur_expir = static_cast<const LCNoncurExpiration_S3&>(noncur_expiration);
       noncur_expir.dump_xml(f);
     }
-    if (!mp_expiration.empty()) {
+    if (mp_expiration.has_days()) {
       const LCMPExpiration_S3& mp_expir = static_cast<const LCMPExpiration_S3&>(mp_expiration);
       mp_expir.dump_xml(f);
     }