rgw_lc: add support for optional filter argument and make ID optional

Support Filter tag in Lifecycle XML similar to AWS S3, while S3 docs
mention that this tag is mandatory, older clients still default to
Prefix, and S3 itself seems to relaxed in enforcing the rule, this
implementation also follows a similar pattern. Filter optionally
supports filtering via (multiple) Object Tags, this is still a TODO. The
current implementation of object tags is still as an object xattr, and
since the LC processing still iterates over the bucket index which
currently doesn't have any info. on tags, this requires some thought
into for implementing without a larger performance penalty

Fixes: http://tracker.ceph.com/issues/20872
Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
(cherry picked from commit 93a858392a2a3bc8c16369dd8c3f6845e99af404)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 2990bff1e816c1fd5269659db91dc08f3f0ea64c..cad0304e3527cb15e5f5b8f1bed0fa9f21f0e077 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -67,7 +67,14 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
     op.mp_expiration = rule->get_mp_expiration().get_days();
   }
   op.dm_expiration = rule->get_dm_expiration();
-  auto ret = prefix_map.insert(pair<string, lc_op>(rule->get_prefix(), op));
+
+  std::string prefix;
+  if (rule->get_filter().has_prefix()){
+    prefix = rule->get_filter().get_prefix();
+  } else {
+    prefix = rule->get_prefix();
+  }
+  auto ret = prefix_map.emplace(std::move(prefix), std::move(op));
   return ret.second;
 }
 

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index dd6a1a7f04664f048cd12962cf2aca6accad507c..0a655c5c3e32a1d651052953befc55853d19a945 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -91,6 +91,47 @@ public:
 };
 WRITE_CLASS_ENCODER(LCExpiration)
 
+class LCFilter
+{
+ protected:
+  std::string prefix;
+  // TODO add support for tagging
+ public:
+  const std::string& get_prefix() const{
+    return prefix;
+  }
+
+  void set_prefix(const string& _prefix){
+    prefix = _prefix;
+  }
+
+  void set_prefix(std::string&& _prefix){
+    prefix = std::move(_prefix);
+  }
+
+  bool empty() const {
+    return prefix.empty();
+  }
+
+  bool has_prefix() const {
+    return !prefix.empty();
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(prefix, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(prefix, bl);
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(LCFilter);
+
+
+
 class LCRule
 {
 protected:
@@ -100,6 +141,7 @@ protected:
   LCExpiration expiration;
   LCExpiration noncur_expiration;
   LCExpiration mp_expiration;
+  LCFilter filter;
   bool dm_expiration = false;
 
 public:
@@ -115,11 +157,15 @@ public:
   string& get_status() {
       return status;
   }
-  
+
   string& get_prefix() {
       return prefix;
   }
 
+  LCFilter& get_filter() {
+    return filter;
+  }
+
   LCExpiration& get_expiration() {
     return expiration;
   }
@@ -167,7 +213,7 @@ public:
   bool valid();
   
   void encode(bufferlist& bl) const {
-     ENCODE_START(4, 1, bl);
+     ENCODE_START(5, 1, bl);
      ::encode(id, bl);
      ::encode(prefix, bl);
      ::encode(status, bl);
@@ -175,10 +221,11 @@ public:
      ::encode(noncur_expiration, bl);
      ::encode(mp_expiration, bl);
      ::encode(dm_expiration, bl);
+     ::encode(filter, bl);
      ENCODE_FINISH(bl);
    }
    void decode(bufferlist::iterator& bl) {
-     DECODE_START_LEGACY_COMPAT_LEN(4, 1, 1, bl);
+     DECODE_START_LEGACY_COMPAT_LEN(5, 1, 1, bl);
      ::decode(id, bl);
      ::decode(prefix, bl);
      ::decode(status, bl);
@@ -192,6 +239,9 @@ public:
      if (struct_v >= 4) {
         ::decode(dm_expiration, bl);
      }
+     if (struct_v >= 5) {
+       ::decode(filter, bl);
+     }
      DECODE_FINISH(bl);
    }
 

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index ea64847a0c733af1ca0eef4583cce7a3812fdc95..b03c4c32b9efa5d9b19397de49066a023a156750 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -80,15 +80,41 @@ bool LCRule_S3::xml_end(const char *el) {
   status.clear();
   dm_expiration = false;
 
+  // S3 generates a 48 bit random ID, maybe we could generate shorter IDs
+  static constexpr auto LC_ID_LENGTH = 48;
+
   lc_id = static_cast<LCID_S3 *>(find_first("ID"));
-  if (!lc_id)
-    return false;
-  id = lc_id->get_data();
+  if (lc_id){
+    id = lc_id->get_data();
+  } else {
+    gen_rand_alphanumeric_lower(nullptr, &id, LC_ID_LENGTH);
+  }
+
+
+  XMLObj *obj = find_first("Filter");
+
+  if (obj){
+    string _prefix;
+    RGWXMLDecoder::decode_xml("Prefix", _prefix, obj);
+    filter.set_prefix(std::move(_prefix));
+  } else {
+    // Ideally the following code should be deprecated and we should return
+    // False here, The new S3 LC configuration xml spec. makes Filter mandatory
+    // and Prefix optional. However older clients including boto2 still generate
+    // xml according to the older spec, where Prefix existed outside of Filter
+    // and S3 itself seems to be sloppy on enforcing the mandatory Filter
+    // argument. A day will come when S3 enforces their own xml-spec, but it is
+    // not this day
+
+    lc_prefix = static_cast<LCPrefix_S3 *>(find_first("Prefix"));
+
+    if (!lc_prefix){
+      return false;
+    }
+
+    prefix = lc_prefix->get_data();
+  }
 
-  lc_prefix = static_cast<LCPrefix_S3 *>(find_first("Prefix"));
-  if (!lc_prefix)
-    return false;
-  prefix = lc_prefix->get_data();
 
   lc_status = static_cast<LCStatus_S3 *>(find_first("Status"));
   if (!lc_status)
@@ -126,7 +152,12 @@ bool LCRule_S3::xml_end(const char *el) {
 void LCRule_S3::to_xml(CephContext *cct, ostream& out) {
   out << "<Rule>" ;
   out << "<ID>" << id << "</ID>";
-  out << "<Prefix>" << prefix << "</Prefix>";
+  if (!filter.empty()) {
+    LCFilter_S3& lc_filter = static_cast<LCFilter_S3&>(filter);
+    lc_filter.to_xml(out);
+  } else {
+    out << "<Prefix>" << prefix << "</Prefix>";
+  }
   out << "<Status>" << status << "</Status>";
   if (!expiration.empty() || dm_expiration) {
     LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);

diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h
index ed1af0c0053e3527d3ec27ac96742cf6e88c26b8..1ac30d8efe8999bff1c429c107b647e87c96e424 100644 (file)
--- a/src/rgw/rgw_lc_s3.h
+++ b/src/rgw/rgw_lc_s3.h
@@ -26,6 +26,25 @@ public:
   string& to_str() { return data; }
 };
 
+class LCFilter_S3 : public LCFilter, public XMLObj
+{
+ public:
+  ~LCFilter_S3() override {}
+  string& to_str() { return data; }
+  void to_xml(ostream& out){
+    out << "<Filter>";
+      if (!prefix.empty())
+        out << "<Prefix>" << prefix << "<Prefix>";
+    out << "</Filter>";
+  }
+  void dump_xml(Formatter *f) const {
+    f->open_object_section("Filter");
+    if (!prefix.empty())
+      encode_xml("Prefix", prefix, f);
+    f->close_section(); // Filter
+  }
+};
+
 class LCStatus_S3 : public XMLObj
 {
 public:
@@ -150,7 +169,13 @@ public:
   void dump_xml(Formatter *f) const {
     f->open_object_section("Rule");
     encode_xml("ID", id, f);
-    encode_xml("Prefix", prefix, f);
+    // In case of an empty filter and an empty Prefix, we defer to Prefix.
+    if (!filter.empty()) {
+      const LCFilter_S3& lc_filter = static_cast<const LCFilter_S3&>(filter);
+      lc_filter.dump_xml(f);
+    } else {
+      encode_xml("Prefix", prefix, f);
+    }
     encode_xml("Status", status, f);
     if (!expiration.empty() || dm_expiration) {
       LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
@@ -164,6 +189,7 @@ public:
       const LCMPExpiration_S3& mp_expir = static_cast<const LCMPExpiration_S3&>(mp_expiration);
       mp_expir.dump_xml(f);
     }
+
     f->close_section(); // Rule
   }
 };