From: Abhishek Lekshmanan Date: Thu, 3 Aug 2017 13:14:05 +0000 (+0200) Subject: rgw_lc: add support for optional filter argument and make ID optional X-Git-Tag: v12.2.1~35^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b575d00073ccf3f84d84a220f542058471a58472;p=ceph.git rgw_lc: add support for optional filter argument and make ID optional Support Filter tag in Lifecycle XML similar to AWS S3, while S3 docs mention that this tag is mandatory, older clients still default to Prefix, and S3 itself seems to relaxed in enforcing the rule, this implementation also follows a similar pattern. Filter optionally supports filtering via (multiple) Object Tags, this is still a TODO. The current implementation of object tags is still as an object xattr, and since the LC processing still iterates over the bucket index which currently doesn't have any info. on tags, this requires some thought into for implementing without a larger performance penalty Fixes: http://tracker.ceph.com/issues/20872 Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 93a858392a2a3bc8c16369dd8c3f6845e99af404) --- diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 2990bff1e816..cad0304e3527 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -67,7 +67,14 @@ bool RGWLifecycleConfiguration::_add_rule(LCRule *rule) op.mp_expiration = rule->get_mp_expiration().get_days(); } op.dm_expiration = rule->get_dm_expiration(); - auto ret = prefix_map.insert(pair(rule->get_prefix(), op)); + + std::string prefix; + if (rule->get_filter().has_prefix()){ + prefix = rule->get_filter().get_prefix(); + } else { + prefix = rule->get_prefix(); + } + auto ret = prefix_map.emplace(std::move(prefix), std::move(op)); return ret.second; } diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index dd6a1a7f0466..0a655c5c3e32 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -91,6 +91,47 @@ public: }; WRITE_CLASS_ENCODER(LCExpiration) +class LCFilter +{ + protected: + std::string prefix; + // TODO add support for tagging + public: + const std::string& get_prefix() const{ + return prefix; + } + + void set_prefix(const string& _prefix){ + prefix = _prefix; + } + + void set_prefix(std::string&& _prefix){ + prefix = std::move(_prefix); + } + + bool empty() const { + return prefix.empty(); + } + + bool has_prefix() const { + return !prefix.empty(); + } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(prefix, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(prefix, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(LCFilter); + + + class LCRule { protected: @@ -100,6 +141,7 @@ protected: LCExpiration expiration; LCExpiration noncur_expiration; LCExpiration mp_expiration; + LCFilter filter; bool dm_expiration = false; public: @@ -115,11 +157,15 @@ public: string& get_status() { return status; } - + string& get_prefix() { return prefix; } + LCFilter& get_filter() { + return filter; + } + LCExpiration& get_expiration() { return expiration; } @@ -167,7 +213,7 @@ public: bool valid(); void encode(bufferlist& bl) const { - ENCODE_START(4, 1, bl); + ENCODE_START(5, 1, bl); ::encode(id, bl); ::encode(prefix, bl); ::encode(status, bl); @@ -175,10 +221,11 @@ public: ::encode(noncur_expiration, bl); ::encode(mp_expiration, bl); ::encode(dm_expiration, bl); + ::encode(filter, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN(4, 1, 1, bl); + DECODE_START_LEGACY_COMPAT_LEN(5, 1, 1, bl); ::decode(id, bl); ::decode(prefix, bl); ::decode(status, bl); @@ -192,6 +239,9 @@ public: if (struct_v >= 4) { ::decode(dm_expiration, bl); } + if (struct_v >= 5) { + ::decode(filter, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc index ea64847a0c73..b03c4c32b9ef 100644 --- a/src/rgw/rgw_lc_s3.cc +++ b/src/rgw/rgw_lc_s3.cc @@ -80,15 +80,41 @@ bool LCRule_S3::xml_end(const char *el) { status.clear(); dm_expiration = false; + // S3 generates a 48 bit random ID, maybe we could generate shorter IDs + static constexpr auto LC_ID_LENGTH = 48; + lc_id = static_cast(find_first("ID")); - if (!lc_id) - return false; - id = lc_id->get_data(); + if (lc_id){ + id = lc_id->get_data(); + } else { + gen_rand_alphanumeric_lower(nullptr, &id, LC_ID_LENGTH); + } + + + XMLObj *obj = find_first("Filter"); + + if (obj){ + string _prefix; + RGWXMLDecoder::decode_xml("Prefix", _prefix, obj); + filter.set_prefix(std::move(_prefix)); + } else { + // Ideally the following code should be deprecated and we should return + // False here, The new S3 LC configuration xml spec. makes Filter mandatory + // and Prefix optional. However older clients including boto2 still generate + // xml according to the older spec, where Prefix existed outside of Filter + // and S3 itself seems to be sloppy on enforcing the mandatory Filter + // argument. A day will come when S3 enforces their own xml-spec, but it is + // not this day + + lc_prefix = static_cast(find_first("Prefix")); + + if (!lc_prefix){ + return false; + } + + prefix = lc_prefix->get_data(); + } - lc_prefix = static_cast(find_first("Prefix")); - if (!lc_prefix) - return false; - prefix = lc_prefix->get_data(); lc_status = static_cast(find_first("Status")); if (!lc_status) @@ -126,7 +152,12 @@ bool LCRule_S3::xml_end(const char *el) { void LCRule_S3::to_xml(CephContext *cct, ostream& out) { out << "" ; out << "" << id << ""; - out << "" << prefix << ""; + if (!filter.empty()) { + LCFilter_S3& lc_filter = static_cast(filter); + lc_filter.to_xml(out); + } else { + out << "" << prefix << ""; + } out << "" << status << ""; if (!expiration.empty() || dm_expiration) { LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration); diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h index ed1af0c0053e..1ac30d8efe89 100644 --- a/src/rgw/rgw_lc_s3.h +++ b/src/rgw/rgw_lc_s3.h @@ -26,6 +26,25 @@ public: string& to_str() { return data; } }; +class LCFilter_S3 : public LCFilter, public XMLObj +{ + public: + ~LCFilter_S3() override {} + string& to_str() { return data; } + void to_xml(ostream& out){ + out << ""; + if (!prefix.empty()) + out << "" << prefix << ""; + out << ""; + } + void dump_xml(Formatter *f) const { + f->open_object_section("Filter"); + if (!prefix.empty()) + encode_xml("Prefix", prefix, f); + f->close_section(); // Filter + } +}; + class LCStatus_S3 : public XMLObj { public: @@ -150,7 +169,13 @@ public: void dump_xml(Formatter *f) const { f->open_object_section("Rule"); encode_xml("ID", id, f); - encode_xml("Prefix", prefix, f); + // In case of an empty filter and an empty Prefix, we defer to Prefix. + if (!filter.empty()) { + const LCFilter_S3& lc_filter = static_cast(filter); + lc_filter.dump_xml(f); + } else { + encode_xml("Prefix", prefix, f); + } encode_xml("Status", status, f); if (!expiration.empty() || dm_expiration) { LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration); @@ -164,6 +189,7 @@ public: const LCMPExpiration_S3& mp_expir = static_cast(mp_expiration); mp_expir.dump_xml(f); } + f->close_section(); // Rule } };