From: Matt Benjamin Date: Sat, 20 Nov 2021 18:45:51 +0000 (-0500) Subject: rgwlc: introduce lifecycle config flags extension X-Git-Tag: v18.0.0~475^2~7 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=23a588da12aa498bb40e800ab1cb84656f2fc763;p=ceph.git rgwlc: introduce lifecycle config flags extension rgwlc: add uint32_t flags bitmap to LCFilter This is intended to support a concise set of extensions to S3 LifecycleConfiguration, initially, just a flag that indicates a rule is intended for execution on RGW ArchiveZone. rgwlc: add machinery to define and recognize LCFilter flags Add a concept of filter flags to lifecycle filter rules, an RGW extension. The initial purpose of flags is to permit marking specific lifecycle rules as specific to an RGW archive zone, but other flags could be added in future. rgwlc: add new unittest_rgw_lc to run internal checks, add a few valid and invalid lifecycle configuration xml parses for now. Fixes: https://tracker.ceph.com/issues/53361 Signed-off-by: Matt Benjamin --- diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 57f412559bc1e..14e64bc952a43 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -17,6 +17,7 @@ #include "include/function2.hpp" #include "common/Formatter.h" #include "common/containers.h" +#include "common/split.h" #include #include "include/random.h" #include "cls/lock/cls_lock_client.h" @@ -55,6 +56,31 @@ const char* LC_STATUS[] = { using namespace librados; +static inline std::string_view sv_trim(std::string_view str) { + while (isspace(str.front())) { + str.remove_prefix(1); + } + while (isspace(str.back())) { + str.remove_suffix(1); + } + return str; +} + +uint32_t LCFilter::recognize_flags(const std::string& flag_expr) +{ + uint32_t flags = 0; + ceph::split sp_flags(flag_expr); // default separators are ,;=\t\n + for (auto it = sp_flags.begin(); it != sp_flags.end(); ++it) { + auto token = sv_trim(string_view{*it}); + for (auto& flag_tok : LCFilter::filter_flags) { + if (token == flag_tok.name) { + flags |= LCFilter::make_flag(flag_tok.bit); + } + } + } + return flags; +} + bool LCRule::valid() const { if (id.length() > MAX_ID_LEN) { diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index e62ab480f8197..81294bc95f332 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -5,6 +5,7 @@ #define CEPH_RGW_LC_H #include +#include #include #include @@ -159,13 +160,51 @@ public: }; WRITE_CLASS_ENCODER(LCTransition) +enum class LCFlagType : uint16_t +{ + none = 0, + ArchiveZone, +}; + +class LCFlag { +public: + LCFlagType bit; + const char* name; + + constexpr LCFlag(LCFlagType ord, const char* name) : bit(ord), name(name) + {} +}; + class LCFilter { - protected: + public: + + static constexpr uint32_t make_flag(LCFlagType type) { + switch (type) { + case LCFlagType::none: + return 0; + break; + default: + return 1 << (uint32_t(type) - 1); + } + } + + static constexpr std::array filter_flags = + { + LCFlag(LCFlagType::none, "none"), + LCFlag(LCFlagType::ArchiveZone, "ArchiveZone"), + }; + +protected: std::string prefix; RGWObjTags obj_tags; + uint32_t flags; - public: +public: +static uint32_t recognize_flags(const std::string& flag_expr); + + LCFilter() : flags(make_flag(LCFlagType::none)) + {} const std::string& get_prefix() const { return prefix; @@ -175,6 +214,10 @@ class LCFilter return obj_tags; } + const uint32_t get_flags() { + return flags; + } + bool empty() const { return !(has_prefix() || has_tags()); } @@ -195,16 +238,20 @@ class LCFilter } void encode(bufferlist& bl) const { - ENCODE_START(2, 1, bl); + ENCODE_START(3, 1, bl); encode(prefix, bl); encode(obj_tags, bl); + encode(flags, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(2, bl); + DECODE_START(3, bl); decode(prefix, bl); if (struct_v >= 2) { decode(obj_tags, bl); + if (struct_v >= 3) { + decode(flags, bl); + } } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc index 536323d74821c..4b966238e823b 100644 --- a/src/rgw/rgw_lc_s3.cc +++ b/src/rgw/rgw_lc_s3.cc @@ -139,15 +139,22 @@ void LCFilter_S3::decode_xml(XMLObj *obj) * Empty filters are allowed: * https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html */ - XMLObj *o = obj->find_first("And"); + XMLObj* o = obj->find_first("And"); if (o == nullptr){ o = obj; } RGWXMLDecoder::decode_xml("Prefix", prefix, o); + + /* parse optional flags (extension) */ + auto flags_iter = o->find("Flag"); + while (auto flag_xml = flags_iter.get_next()){ + flags |= LCFilter::recognize_flags(flag_xml->get_data()); + } + + obj_tags.clear(); // why is this needed? auto tags_iter = o->find("Tag"); - obj_tags.clear(); - while (auto tag_xml =tags_iter.get_next()){ + while (auto tag_xml = tags_iter.get_next()){ std::string _key,_val; RGWXMLDecoder::decode_xml("Key", _key, tag_xml); RGWXMLDecoder::decode_xml("Value", _val, tag_xml); diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h index 0fe093df9ea0f..84ffdc6c8ad83 100644 --- a/src/rgw/rgw_lc_s3.h +++ b/src/rgw/rgw_lc_s3.h @@ -14,6 +14,7 @@ #include "rgw_xml.h" #include "rgw_tag_s3.h" + class LCFilter_S3 : public LCFilter { public: diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index 374ea0bd41527..55921737665d5 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -194,6 +194,14 @@ add_ceph_unittest(unittest_rgw_xml) target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES}) +# unittest_rgw_lc +add_executable(unittest_rgw_lc test_rgw_lc.cc) +add_ceph_unittest(unittest_rgw_lc) +target_include_directories(unittest_rgw_lc SYSTEM PRIVATE + "${CMAKE_SOURCE_DIR}/src/rgw") +target_link_libraries(unittest_rgw_lc + rgw_common ${rgw_libs} ${EXPAT_LIBRARIES}) + # unittest_rgw_arn add_executable(unittest_rgw_arn test_rgw_arn.cc) add_ceph_unittest(unittest_rgw_arn) diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc new file mode 100644 index 0000000000000..2ff992862003c --- /dev/null +++ b/src/test/rgw/test_rgw_lc.cc @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "rgw_xml.h" +#include "rgw_lc.h" +#include "rgw_lc_s3.h" +#include +//#include +#include +#include +#include + +static const char* xmldoc_1 = +R"( + + tax/ + + key1 + value1 + + + key2 + value2 + + + +)"; + +TEST(TestLCFilterDecoder, XMLDoc1) +{ + RGWXMLDecoder::XMLParser parser; + ASSERT_TRUE(parser.init()); + ASSERT_TRUE(parser.parse(xmldoc_1, strlen(xmldoc_1), 1)); + LCFilter_S3 filter; + auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true); + ASSERT_TRUE(result); + /* check repeated Tag element */ + auto tag_map = filter.get_tags().get_tags(); + auto val1 = tag_map.find("key1"); + ASSERT_EQ(val1->second, "value1"); + auto val2 = tag_map.find("key2"); + ASSERT_EQ(val2->second, "value2"); + /* check our flags */ + ASSERT_EQ(filter.get_flags(), 0); +} + +static const char* xmldoc_2 = +R"( + + + ArchiveZone + + + ArchiveZone + + + spongebob + squarepants + + + +)"; + +TEST(TestLCFilterDecoder, XMLDoc2) +{ + RGWXMLDecoder::XMLParser parser; + ASSERT_TRUE(parser.init()); + ASSERT_TRUE(parser.parse(xmldoc_2, strlen(xmldoc_2), 1)); + LCFilter_S3 filter; + auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true); + ASSERT_TRUE(result); + /* check repeated Tag element */ + auto tag_map = filter.get_tags().get_tags(); + auto val1 = tag_map.find("spongebob"); + ASSERT_EQ(val1->second, "squarepants"); + /* check our flags */ + ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::ArchiveZone)); +} + +// invalid And element placement +static const char* xmldoc_3 = +R"( + + + miles + davis + + + + spongebob + squarepants + + +)"; + +TEST(TestLCFilterInvalidAnd, XMLDoc3) +{ + RGWXMLDecoder::XMLParser parser; + ASSERT_TRUE(parser.init()); + ASSERT_TRUE(parser.parse(xmldoc_3, strlen(xmldoc_3), 1)); + LCFilter_S3 filter; + auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true); + ASSERT_TRUE(result); + /* check repeated Tag element */ + auto tag_map = filter.get_tags().get_tags(); + auto val1 = tag_map.find("spongebob"); + ASSERT_TRUE(val1 == tag_map.end()); + /* because the invalid 2nd tag element was not recognized, + * we cannot access it: + ASSERT_EQ(val1->second, "squarepants"); + */ + /* check our flags */ + ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none)); +}