rgw: add support for delete marker expiration in s3 lifecycle.

Fixes: http://tracker.ceph.com/issues/19730
Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 1b7ce3c572e228691f52ef9594cb72003f03b2eb..e45326c6fe900e514205195f3baede5bbd12c2e8 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -32,7 +32,7 @@ bool LCRule::validate()
   if (id.length() > MAX_ID_LEN) {
     return false;
   }
-  else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty()) {
+  else if(expiration.empty() && noncur_expiration.empty() && mp_expiration.empty() && !dm_expiration) {
     return false;
   }
   else if (!expiration.empty() && expiration.get_days() <= 0) {
@@ -69,6 +69,7 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
   if (!rule->get_mp_expiration().empty()) {
     op.mp_expiration = rule->get_mp_expiration().get_days();
   }
+  op.dm_expiration = rule->get_dm_expiration();
   auto ret = prefix_map.insert(pair<string, lc_op>(rule->get_prefix(), op));
   return ret.second;
 }
@@ -228,17 +229,16 @@ int RGWLC::bucket_lc_prepare(int index)
 
 bool RGWLC::obj_has_expired(double timediff, int days)
 {
-       double cmp;
-
-       if (cct->_conf->rgw_lc_debug_interval <= 0) {
-               /* Normal case, run properly */
-               cmp = days*24*60*60;
-       } else {
-               /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */
-               cmp = days*cct->_conf->rgw_lc_debug_interval;
-       }
+  double cmp;
+  if (cct->_conf->rgw_lc_debug_interval <= 0) {
+    /* Normal case, run properly */
+    cmp = days*24*60*60;
+  } else {
+    /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */
+    cmp = days*cct->_conf->rgw_lc_debug_interval;
+  }
 
-       return (timediff >= cmp);
+  return (timediff >= cmp);
 }
 
 int RGWLC::remove_expired_obj(RGWBucketInfo& bucket_info, rgw_obj_key obj_key, bool remove_indeed)
@@ -395,7 +395,8 @@ int RGWLC::bucket_lc_process(string& shard_id)
   //bucket versioning is enabled or suspended
     rgw_obj_key pre_marker;
     for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
-      if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 && prefix_iter->second.noncur_expiration <= 0)) {
+      if (!prefix_iter->second.status || (prefix_iter->second.expiration <= 0 
+        && prefix_iter->second.noncur_expiration <= 0 && !prefix_iter->second.dm_expiration)) {
         continue;
       }
       if (prefix_iter != prefix_map.begin() && 
@@ -425,9 +426,11 @@ int RGWLC::bucket_lc_process(string& shard_id)
         ceph::real_time mtime;
         bool remove_indeed = true;
         int expiration;
+        bool skip_expiration;
         for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
+          skip_expiration = false;
           if (obj_iter->is_current()) {
-            if (prefix_iter->second.expiration <= 0) {
+            if (prefix_iter->second.expiration <= 0 && !prefix_iter->second.dm_expiration) {
               continue;
             }
             if (obj_iter->is_delete_marker()) {
@@ -440,12 +443,16 @@ int RGWLC::bucket_lc_process(string& shard_id)
               } else if (obj_iter->key.name.compare((obj_iter + 1)->key.name) == 0) {   //*obj_iter is delete marker and isn't the only version, do nothing.
                 continue;
               }
+              skip_expiration = prefix_iter->second.dm_expiration;
               remove_indeed = true;   //we should remove the delete marker if it's the only version
             } else {
               remove_indeed = false;
             }
             mtime = obj_iter->meta.mtime;
             expiration = prefix_iter->second.expiration;
+            if (!skip_expiration && expiration <= 0) {
+              continue;
+            }
           } else {
             if (prefix_iter->second.noncur_expiration <=0) {
               continue;
@@ -454,7 +461,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
             mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime;
             expiration = prefix_iter->second.noncur_expiration;
           }
-          if (obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
+          if (skip_expiration || obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
             if (obj_iter->is_visible()) {
               RGWObjectCtx rctx(store);
               rgw_obj obj(bucket_info.bucket, obj_iter->key);

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index 480ba3fe513bbab9b87913b74f16a603aa5ea59e..9609c952c45e93a07bb23a5b291c4bdc0ef77ac9 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -55,6 +55,9 @@ public:
   void dump(Formatter *f) const;
 //  static void generate_test_instances(list<ACLOwner*>& o);
   void set_days(const string& _days) { days = _days; }
+  string get_days_str() const{
+    return days;
+  }
   int get_days() {return atoi(days.c_str()); }
   bool empty() const{
     return days.empty();
@@ -71,6 +74,7 @@ protected:
   LCExpiration expiration;
   LCExpiration noncur_expiration;
   LCExpiration mp_expiration;
+  bool dm_expiration;
 
 public:
 
@@ -102,6 +106,10 @@ public:
     return mp_expiration;
   }
 
+  bool get_dm_expiration() {
+    return dm_expiration;
+  }
+
   void set_id(string*_id) {
     id = *_id;
   }
@@ -126,20 +134,25 @@ public:
     mp_expiration = *_mp_expiration;
   }
 
+  void set_dm_expiration(bool _dm_expiration) {
+    dm_expiration = _dm_expiration;
+  }
+
   bool validate();
   
   void encode(bufferlist& bl) const {
-     ENCODE_START(3, 1, bl);
+     ENCODE_START(4, 1, bl);
      ::encode(id, bl);
      ::encode(prefix, bl);
      ::encode(status, bl);
      ::encode(expiration, bl);
      ::encode(noncur_expiration, bl);
      ::encode(mp_expiration, bl);
+     ::encode(dm_expiration, bl);
      ENCODE_FINISH(bl);
    }
    void decode(bufferlist::iterator& bl) {
-     DECODE_START_LEGACY_COMPAT_LEN(3, 1, 1, bl);
+     DECODE_START_LEGACY_COMPAT_LEN(4, 1, 1, bl);
      ::decode(id, bl);
      ::decode(prefix, bl);
      ::decode(status, bl);
@@ -150,6 +163,9 @@ public:
      if (struct_v >= 3) {
        ::decode(mp_expiration, bl);
      }
+     if (struct_v >= 4) {
+        ::decode(dm_expiration, bl);
+     }
      DECODE_FINISH(bl);
    }
 
@@ -159,11 +175,12 @@ WRITE_CLASS_ENCODER(LCRule)
 struct lc_op
 {
   bool status;
+  bool dm_expiration;
   int expiration;
   int noncur_expiration;
   int mp_expiration;
 
-  lc_op() : status(false), expiration(0), noncur_expiration(0), mp_expiration(0) {}
+  lc_op() : status(false), dm_expiration(false), expiration(0), noncur_expiration(0), mp_expiration(0) {}
   
 };
 

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index 99ed7b70cd688a708f01f15add19ab92e1cc52c3..eb58a0c9f4108c5c76e59f76a901d681f0d8b66f 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -15,10 +15,19 @@ using namespace std;
 
 bool LCExpiration_S3::xml_end(const char * el) {
   LCDays_S3 *lc_days = static_cast<LCDays_S3 *>(find_first("Days"));
+  LCDeleteMarker_S3 *lc_dm = static_cast<LCDeleteMarker_S3 *>(find_first("ExpiredObjectDeleteMarker"));
 
-  if (!lc_days)
+  if ((!lc_days && !lc_dm) || (lc_days && lc_dm)) {
     return false;
-  days = lc_days->get_data();
+  }
+  if (lc_days) {
+    days = lc_days->get_data();
+  } else if (lc_dm) {
+    dm_expiration = lc_dm->get_data().compare("true") == 0;
+    if (!dm_expiration) {
+      return false;
+    }
+  }
   return true;
 }
 
@@ -61,6 +70,7 @@ bool LCRule_S3::xml_end(const char *el) {
   id.clear();
   prefix.clear();
   status.clear();
+  dm_expiration = false;
 
   lc_id = static_cast<LCID_S3 *>(find_first("ID"));
   if (!lc_id)
@@ -86,7 +96,11 @@ bool LCRule_S3::xml_end(const char *el) {
     return false;
   } else {
     if (lc_expiration) {
-      expiration = *lc_expiration;
+      if (!lc_expiration->empty()) {
+        expiration.set_days(lc_expiration->get_days_str());
+      } else {
+        dm_expiration = lc_expiration->get_dm_expiration();
+      }
     }
     if (lc_noncur_expiration) {
       noncur_expiration = *lc_noncur_expiration;
@@ -104,8 +118,8 @@ void LCRule_S3::to_xml(CephContext *cct, ostream& out) {
   out << "<ID>" << id << "</ID>";
   out << "<Prefix>" << prefix << "</Prefix>";
   out << "<Status>" << status << "</Status>";
-  if (!expiration.empty()) {
-    LCExpiration_S3& expir = static_cast<LCExpiration_S3&>(expiration);
+  if (!expiration.empty() || dm_expiration) {
+    LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
     expir.to_xml(out);
   }
   if (!noncur_expiration.empty()) {
@@ -164,6 +178,8 @@ XMLObj *RGWLCXMLParser_S3::alloc_obj(const char *el)
     obj = new LCExpiration_S3();
   } else if (strcmp(el, "Days") == 0) {
     obj = new LCDays_S3();
+  } else if (strcmp(el, "ExpiredObjectDeleteMarker") == 0) {
+    obj = new LCDeleteMarker_S3();
   } else if (strcmp(el, "NoncurrentVersionExpiration") == 0) {
     obj = new LCNoncurExpiration_S3();
   } else if (strcmp(el, "NoncurrentDays") == 0) {

diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h
index 52f6987e4877541257aa6f98a2866acb773535ae..53c5f7e8ba59eeaefc83d17f336a1f35c2a863c3 100644 (file)
--- a/src/rgw/rgw_lc_s3.h
+++ b/src/rgw/rgw_lc_s3.h
@@ -42,20 +42,50 @@ public:
   string& to_str() { return data; }
 };
 
+class LCDeleteMarker_S3 : public XMLObj
+{
+public:
+  LCDeleteMarker_S3() {}
+  ~LCDeleteMarker_S3() override {}
+  string& to_str() { return data; }
+};
+
 class LCExpiration_S3 : public LCExpiration, public XMLObj
 {
+private:
+  bool dm_expiration;
 public:
-  LCExpiration_S3() {}
+  LCExpiration_S3(): dm_expiration(false) {}
+  LCExpiration_S3(string _days, bool _dm_expiration) {
+    days = _days;
+    dm_expiration = _dm_expiration;
+  }
   ~LCExpiration_S3() override {}
 
   bool xml_end(const char *el) override;
   void to_xml(ostream& out) {
-    out << "<Expiration>" << "<Days>" << days << "</Days>"<< "</Expiration>";
+    if (dm_expiration) {
+      out << "<Expiration>" << "<ExpiredObjectDeleteMarker>" << "true" << "</ExpiredObjectDeleteMarker>" << "</Expiration>";
+    } else {
+      out << "<Expiration>" << "<Days>" << days << "</Days>"<< "</Expiration>";
+    }
   }
   void dump_xml(Formatter *f) const {
-         f->open_object_section("Expiration");
-         encode_xml("Days", days, f);
-         f->close_section(); // Expiration
+    f->open_object_section("Expiration");
+    if (dm_expiration) {
+      encode_xml("ExpiredObjectDeleteMarker", "true", f);
+    } else {
+      encode_xml("Days", days, f);
+    }
+    f->close_section(); // Expiration
+  }
+
+  void set_dm_expiration(bool _dm_expiration) {
+    dm_expiration = _dm_expiration;
+  }
+
+  bool get_dm_expiration() {
+    return dm_expiration;
   }
 };
 
@@ -107,8 +137,8 @@ public:
     encode_xml("ID", id, f);
     encode_xml("Prefix", prefix, f);
     encode_xml("Status", status, f);
-    if (!expiration.empty()) {
-      const LCExpiration_S3& expir = static_cast<const LCExpiration_S3&>(expiration);
+    if (!expiration.empty() || dm_expiration) {
+      LCExpiration_S3 expir(expiration.get_days_str(), dm_expiration);
       expir.dump_xml(f);
     }
     if (!noncur_expiration.empty()) {