rgwlc: introduce lifecycle config flags extension

rgwlc: add uint32_t flags bitmap to LCFilter

This is intended to support a concise set of extensions to S3
LifecycleConfiguration, initially, just a flag that indicates a
rule is intended for execution on RGW ArchiveZone.

rgwlc: add machinery to define and recognize LCFilter flags

Add a concept of filter flags to lifecycle filter rules, an RGW
extension.  The initial purpose of flags is to permit marking
specific lifecycle rules as specific to an RGW archive zone, but
other flags could be added in future.

rgwlc: add new unittest_rgw_lc to run internal checks, add a few
valid and invalid lifecycle configuration xml  parses for now.

Fixes: https://tracker.ceph.com/issues/53361
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 57f412559bc1e3230440387ee96eae9c9f884ba9..14e64bc952a433c686473d1f4dcfa9e5086f7119 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -17,6 +17,7 @@
 #include "include/function2.hpp"
 #include "common/Formatter.h"
 #include "common/containers.h"
+#include "common/split.h"
 #include <common/errno.h>
 #include "include/random.h"
 #include "cls/lock/cls_lock_client.h"
@@ -55,6 +56,31 @@ const char* LC_STATUS[] = {
 
 using namespace librados;
 
+static inline std::string_view sv_trim(std::string_view str) {
+  while (isspace(str.front())) {
+    str.remove_prefix(1);
+  }
+  while (isspace(str.back())) {
+    str.remove_suffix(1);
+  }
+  return str;
+}
+
+uint32_t LCFilter::recognize_flags(const std::string& flag_expr)
+{
+  uint32_t flags = 0;
+  ceph::split sp_flags(flag_expr); // default separators are ,;=\t\n
+  for (auto it = sp_flags.begin(); it != sp_flags.end(); ++it) {
+    auto token = sv_trim(string_view{*it});
+    for (auto& flag_tok : LCFilter::filter_flags) {
+      if (token == flag_tok.name) {
+       flags |= LCFilter::make_flag(flag_tok.bit);
+      }
+    }
+  }
+  return flags;
+}
+
 bool LCRule::valid() const
 {
   if (id.length() > MAX_ID_LEN) {

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index e62ab480f81975a2a624514855d9232a06d456f0..81294bc95f332a23fe1bf276436dec15a09c110b 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -5,6 +5,7 @@
 #define CEPH_RGW_LC_H
 
 #include <map>
+#include <array>
 #include <string>
 #include <iostream>
 
@@ -159,13 +160,51 @@ public:
 };
 WRITE_CLASS_ENCODER(LCTransition)
 
+enum class LCFlagType : uint16_t
+{
+  none = 0,
+  ArchiveZone,
+};
+
+class LCFlag {
+public:
+  LCFlagType bit;
+  const char* name;
+
+  constexpr LCFlag(LCFlagType ord, const char* name) : bit(ord), name(name)
+    {}
+};
+
 class LCFilter
 {
- protected:
+ public:
+
+  static constexpr uint32_t make_flag(LCFlagType type) {
+    switch (type) {
+    case LCFlagType::none:
+      return 0;
+      break;
+    default:
+      return 1 << (uint32_t(type) - 1);
+    }
+   }
+
+  static constexpr std::array<LCFlag, 2> filter_flags =
+  {
+    LCFlag(LCFlagType::none, "none"),
+    LCFlag(LCFlagType::ArchiveZone, "ArchiveZone"),
+  };
+
+protected:
   std::string prefix;
   RGWObjTags obj_tags;
+  uint32_t flags;
 
- public:
+public:
+static uint32_t recognize_flags(const std::string& flag_expr);
+
+  LCFilter() : flags(make_flag(LCFlagType::none))
+    {}
 
   const std::string& get_prefix() const {
     return prefix;
@@ -175,6 +214,10 @@ class LCFilter
     return obj_tags;
   }
 
+  const uint32_t get_flags() {
+    return flags;
+  }
+
   bool empty() const {
     return !(has_prefix() || has_tags());
   }
@@ -195,16 +238,20 @@ class LCFilter
   }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(2, 1, bl);
+    ENCODE_START(3, 1, bl);
     encode(prefix, bl);
     encode(obj_tags, bl);
+    encode(flags, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(2, bl);
+    DECODE_START(3, bl);
     decode(prefix, bl);
     if (struct_v >= 2) {
       decode(obj_tags, bl);
+      if (struct_v >= 3) {
+       decode(flags, bl);
+      }
     }
     DECODE_FINISH(bl);
   }

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index 536323d74821cc21833f6f0a3a34998cc9d7dab3..4b966238e823bfac55c5d7bf54499c7f7b44799f 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -139,15 +139,22 @@ void LCFilter_S3::decode_xml(XMLObj *obj)
    * Empty filters are allowed:
    * https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html
    */
-  XMLObj *o = obj->find_first("And");
+  XMLObj* o = obj->find_first("And");
   if (o == nullptr){
     o = obj;
   }
 
   RGWXMLDecoder::decode_xml("Prefix", prefix, o);
+
+  /* parse optional flags (extension) */
+  auto flags_iter = o->find("Flag");
+  while (auto flag_xml = flags_iter.get_next()){
+    flags |= LCFilter::recognize_flags(flag_xml->get_data());
+  }
+
+  obj_tags.clear(); // why is this needed?
   auto tags_iter = o->find("Tag");
-  obj_tags.clear();
-  while (auto tag_xml =tags_iter.get_next()){
+  while (auto tag_xml = tags_iter.get_next()){
     std::string _key,_val;
     RGWXMLDecoder::decode_xml("Key", _key, tag_xml);
     RGWXMLDecoder::decode_xml("Value", _val, tag_xml);

diff --git a/src/rgw/rgw_lc_s3.h b/src/rgw/rgw_lc_s3.h
index 0fe093df9ea0fbb6b654c7ed3e16c06f309891a6..84ffdc6c8ad83eca53f7c80d5a732277922fce82 100644 (file)
--- a/src/rgw/rgw_lc_s3.h
+++ b/src/rgw/rgw_lc_s3.h
@@ -14,6 +14,7 @@
 #include "rgw_xml.h"
 #include "rgw_tag_s3.h"
 
+
 class LCFilter_S3 : public LCFilter
 {
 public:

diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
index 374ea0bd4152766b6fca18e06251e6111bb7b1f3..55921737665d59ecbed3bab31c04728d2ca6cea9 100644 (file)
--- a/src/test/rgw/CMakeLists.txt
+++ b/src/test/rgw/CMakeLists.txt
@@ -194,6 +194,14 @@ add_ceph_unittest(unittest_rgw_xml)
 
 target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES})
 
+# unittest_rgw_lc
+add_executable(unittest_rgw_lc test_rgw_lc.cc)
+add_ceph_unittest(unittest_rgw_lc)
+target_include_directories(unittest_rgw_lc SYSTEM PRIVATE
+  "${CMAKE_SOURCE_DIR}/src/rgw")
+target_link_libraries(unittest_rgw_lc
+  rgw_common ${rgw_libs} ${EXPAT_LIBRARIES})
+
 # unittest_rgw_arn
 add_executable(unittest_rgw_arn test_rgw_arn.cc)
 add_ceph_unittest(unittest_rgw_arn)

diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
new file mode 100644 (file)
index 0000000..2ff9928
--- /dev/null
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw_xml.h"
+#include "rgw_lc.h"
+#include "rgw_lc_s3.h"
+#include <gtest/gtest.h>
+//#include <spawn/spawn.hpp>
+#include <string>
+#include <vector>
+#include <stdexcept>
+
+static const char* xmldoc_1 =
+R"(<Filter>
+   <And>
+      <Prefix>tax/</Prefix>
+      <Tag>
+         <Key>key1</Key>
+         <Value>value1</Value>
+      </Tag>
+      <Tag>
+         <Key>key2</Key>
+         <Value>value2</Value>
+      </Tag>
+    </And>
+</Filter>
+)";
+
+TEST(TestLCFilterDecoder, XMLDoc1)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_1, strlen(xmldoc_1), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("key1");
+  ASSERT_EQ(val1->second, "value1");
+  auto val2 = tag_map.find("key2");
+  ASSERT_EQ(val2->second, "value2");
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), 0);
+}
+
+static const char* xmldoc_2 =
+R"(<Filter>
+   <And>
+      <Flag>
+         ArchiveZone
+      </Flag>
+      <Flag>
+         ArchiveZone
+      </Flag>
+      <Tag>
+         <Key>spongebob</Key>
+         <Value>squarepants</Value>
+      </Tag>
+    </And>
+</Filter>
+)";
+
+TEST(TestLCFilterDecoder, XMLDoc2)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_2, strlen(xmldoc_2), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("spongebob");
+  ASSERT_EQ(val1->second, "squarepants");
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::ArchiveZone));
+}
+
+// invalid And element placement
+static const char* xmldoc_3 =
+R"(<Filter>
+    <And>
+      <Tag>
+         <Key>miles</Key>
+         <Value>davis</Value>
+      </Tag>
+    </And>
+      <Tag>
+         <Key>spongebob</Key>
+         <Value>squarepants</Value>
+      </Tag>
+</Filter>
+)";
+
+TEST(TestLCFilterInvalidAnd, XMLDoc3)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_3, strlen(xmldoc_3), 1));
+  LCFilter_S3 filter;
+  auto result = RGWXMLDecoder::decode_xml("Filter", filter, &parser, true);
+  ASSERT_TRUE(result);
+  /* check repeated Tag element */
+  auto tag_map = filter.get_tags().get_tags();
+  auto val1 = tag_map.find("spongebob");
+  ASSERT_TRUE(val1 == tag_map.end());
+  /* because the invalid 2nd tag element was not recognized,
+   * we cannot access it:
+  ASSERT_EQ(val1->second, "squarepants");
+  */
+  /* check our flags */
+  ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
+}