rgwlc: introduce lifecycle config flags extension

rgwlc: add uint32_t flags bitmap to LCFilter

This is intended to support a concise set of extensions to S3
LifecycleConfiguration, initially, just a flag that indicates a
rule is intended for execution on RGW ArchiveZone.

rgwlc: add machinery to define and recognize LCFilter flags

Add a concept of filter flags to lifecycle filter rules, an RGW
extension.  The initial purpose of flags is to permit marking
specific lifecycle rules as specific to an RGW archive zone, but
other flags could be added in future.

rgwlc: add new unittest_rgw_lc to run internal checks, add a few
valid and invalid lifecycle configuration xml  parses for now.

Fixes: https://tracker.ceph.com/issues/53361
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit 23a588da12aa498bb40e800ab1cb84656f2fc763)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 1a23988f8fc7f014bc7a92f887f0862d58443298..813b1d2e05b0e4e9f1cf7fd3039fa3c3ab99139f 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -16,6 +16,7 @@
 #include "include/scope_guard.h"
 #include "common/Formatter.h"
 #include "common/containers.h"
+#include "common/split.h"
 #include <common/errno.h>
 #include "include/random.h"
 #include "cls/lock/cls_lock_client.h"
@@ -55,6 +56,31 @@ const char* LC_STATUS[] = {
 
 using namespace librados;
 
+static inline std::string_view sv_trim(std::string_view str) {
+  while (isspace(str.front())) {
+    str.remove_prefix(1);
+  }
+  while (isspace(str.back())) {
+    str.remove_suffix(1);
+  }
+  return str;
+}
+
+uint32_t LCFilter::recognize_flags(const std::string& flag_expr)
+{
+  uint32_t flags = 0;
+  ceph::split sp_flags(flag_expr); // default separators are ,;=\t\n
+  for (auto it = sp_flags.begin(); it != sp_flags.end(); ++it) {
+    auto token = sv_trim(string_view{*it});
+    for (auto& flag_tok : LCFilter::filter_flags) {
+      if (token == flag_tok.name) {
+       flags |= LCFilter::make_flag(flag_tok.bit);
+      }
+    }
+  }
+  return flags;
+}
+
 bool LCRule::valid() const
 {
   if (id.length() > MAX_ID_LEN) {

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index f79dd23ae7c565c3a16c04d56f4cffdacfc129e2..e548e18893e735252b888f2418bbc211ef467c90 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -5,6 +5,7 @@
 #define CEPH_RGW_LC_H
 
 #include <map>
+#include <array>
 #include <string>
 #include <iostream>
 
@@ -159,13 +160,51 @@ public:
 };
 WRITE_CLASS_ENCODER(LCTransition)
 
+enum class LCFlagType : uint16_t
+{
+  none = 0,
+  ArchiveZone,
+};
+
+class LCFlag {
+public:
+  LCFlagType bit;
+  const char* name;
+
+  constexpr LCFlag(LCFlagType ord, const char* name) : bit(ord), name(name)
+    {}
+};
+
 class LCFilter
 {
- protected:
+ public:
+
+  static constexpr uint32_t make_flag(LCFlagType type) {
+    switch (type) {
+    case LCFlagType::none:
+      return 0;
+      break;
+    default:
+      return 1 << (uint32_t(type) - 1);
+    }
+   }
+
+  static constexpr std::array<LCFlag, 2> filter_flags =
+  {
+    LCFlag(LCFlagType::none, "none"),
+    LCFlag(LCFlagType::ArchiveZone, "ArchiveZone"),
+  };
+
+protected:
   std::string prefix;
   RGWObjTags obj_tags;
+  uint32_t flags;
 
- public:
+public:
+static uint32_t recognize_flags(const std::string& flag_expr);
+
+  LCFilter() : flags(make_flag(LCFlagType::none))
+    {}
 
   const std::string& get_prefix() const {
     return prefix;
@@ -175,6 +214,10 @@ class LCFilter
     return obj_tags;
   }
 
+  const uint32_t get_flags() {
+    return flags;
+  }
+
   bool empty() const {
     return !(has_prefix() || has_tags());
   }
@@ -195,16 +238,20 @@ class LCFilter
   }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(2, 1, bl);
+    ENCODE_START(3, 1, bl);
     encode(prefix, bl);
     encode(obj_tags, bl);
+    encode(flags, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(2, bl);
+    DECODE_START(3, bl);
     decode(prefix, bl);
     if (struct_v >= 2) {
       decode(obj_tags, bl);
+      if (struct_v >= 3) {
+       decode(flags, bl);
+      }
     }
     DECODE_FINISH(bl);
   }

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index 536323d74821cc21833f6f0a3a34998cc9d7dab3..4b966238e823bfac55c5d7bf54499c7f7b44799f 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -139,15 +139,22 @@ void LCFilter_S3::decode_xml(XMLObj *obj)
    * Empty filters are allowed:
    * https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html
    */
-  XMLObj *o = obj->find_first("And");
+  XMLObj* o = obj->find_first("And");
   if (o == nullptr){
     o = obj;
   }
 
   RGWXMLDecoder::decode_xml("Prefix", prefix, o);
+
+  /* parse optional flags (extension) */
+  auto flags_iter = o->find("Flag");
+  while (auto flag_xml = flags_iter.get_next()){
+    flags |= LCFilter::recognize_flags(flag_xml->get_data());
+  }
+
+  obj_tags.clear(); // why is this needed?
   auto tags_iter = o->find("Tag");
-  obj_tags.clear();
-  while (auto tag_xml =tags_iter.get_next()){
+  while (auto tag_xml = tags_iter.get_next()){
     std::string _key,_val;
     RGWXMLDecoder::decode_xml("Key", _key, tag_xml);
     RGWXMLDecoder::decode_xml("Value", _val, tag_xml);

diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h
index 0fe093df9ea0fbb6b654c7ed3e16c06f309891a6..84ffdc6c8ad83eca53f7c80d5a732277922fce82 100644 (file)
--- a/src/rgw/rgw_lc_s3.h
+++ b/src/rgw/rgw_lc_s3.h
@@ -14,6 +14,7 @@
 #include "rgw_xml.h"
 #include "rgw_tag_s3.h"
 
+
 class LCFilter_S3 : public LCFilter
 {
 public:

diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
index 2df70d88e2ef1b86470b6cb2a25b4633199c391d..5c09e2203ddb07b614d820d590bf6ec666947176 100644 (file)
--- a/src/test/rgw/CMakeLists.txt
+++ b/src/test/rgw/CMakeLists.txt
@@ -190,6 +190,14 @@ add_ceph_unittest(unittest_rgw_xml)
 
 target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES})
 
+# unittest_rgw_lc
+add_executable(unittest_rgw_lc test_rgw_lc.cc)
+add_ceph_unittest(unittest_rgw_lc)
+target_include_directories(unittest_rgw_lc SYSTEM PRIVATE
+  "${CMAKE_SOURCE_DIR}/src/rgw")
+target_link_libraries(unittest_rgw_lc
+  rgw_common ${rgw_libs} ${EXPAT_LIBRARIES})
+
 # unittest_rgw_arn
 add_executable(unittest_rgw_arn test_rgw_arn.cc)
 add_ceph_unittest(unittest_rgw_arn)

diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
new file mode 100644 (file)
index 0000000..2ff9928
--- /dev/null
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw_xml.h"
+#include "rgw_lc.h"
+#include "rgw_lc_s3.h"
+#include <gtest/gtest.h>
+//#include <spawn/spawn.hpp>
+#include <string>
+#include <vector>
+#include <stdexcept>
+
+static const char* xmldoc_1 =
+R"(<Filter>
+   <And>
+      <Prefix>tax/</Prefix>
+      <Tag>
+         <Key>key1</Key>
+         <Value>value1</Value>
+      </Tag>
+      <Tag>
+         <Key>key2</Key>
+         <Value>value2</Value>
+      </Tag>
+    </And>
+</Filter>
+)";
+
+TEST(TestLCFilterDecoder, XMLDoc1)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_1, strlen(xmldoc_1), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("key1");
+  ASSERT_EQ(val1->second, "value1");
+  auto val2 = tag_map.find("key2");
+  ASSERT_EQ(val2->second, "value2");
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), 0);
+}
+
+static const char* xmldoc_2 =
+R"(<Filter>
+   <And>
+      <Flag>
+         ArchiveZone
+      </Flag>
+      <Flag>
+         ArchiveZone
+      </Flag>
+      <Tag>
+         <Key>spongebob</Key>
+         <Value>squarepants</Value>
+      </Tag>
+    </And>
+</Filter>
+)";
+
+TEST(TestLCFilterDecoder, XMLDoc2)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_2, strlen(xmldoc_2), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("spongebob");
+  ASSERT_EQ(val1->second, "squarepants");
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::ArchiveZone));
+}
+
+// invalid And element placement
+static const char* xmldoc_3 =
+R"(<Filter>
+    <And>
+      <Tag>
+         <Key>miles</Key>
+         <Value>davis</Value>
+      </Tag>
+    </And>
+      <Tag>
+         <Key>spongebob</Key>
+         <Value>squarepants</Value>
+      </Tag>
+</Filter>
+)";
+
+TEST(TestLCFilterInvalidAnd, XMLDoc3)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_3, strlen(xmldoc_3), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("spongebob");
+  ASSERT_TRUE(val1 == tag_map.end());
+  /* because the invalid 2nd tag element was not recognized,
+   * we cannot access it:
+  ASSERT_EQ(val1->second, "squarepants");
+  */
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
+}