rgwlc: implement Filter size limit checks

i.e., this implements ObjectSizeGreaterThan and ObjectSizeLessThan
for supported expiration and transition actions.

Fixes: https://tracker.ceph.com/issues/63304
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index feee38d1f75c7d719c4687ff55c61125c4cb83b7..7169e09edbc39d3d7f1128c3472b160f7c5ab56f 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -155,10 +155,17 @@ bool RGWLifecycleConfiguration::_add_rule(const LCRule& rule)
   } else {
     prefix = rule.get_prefix();
   }
-  if (rule.get_filter().has_tags()){
-    op.obj_tags = rule.get_filter().get_tags();
+  const auto& filter = rule.get_filter();
+  if (filter.has_tags()){
+    op.obj_tags = filter.get_tags();
   }
-  op.rule_flags = rule.get_filter().get_flags();
+  if (filter.has_size_gt()) {
+    op.size_gt = filter.get_size_gt();
+  }
+  if (filter.has_size_lt()) {
+    op.size_lt = filter.get_size_lt();
+  }
+  op.rule_flags = filter.get_flags();
   prefix_map.emplace(std::move(prefix), std::move(op));
   return true;
 }
@@ -262,8 +269,9 @@ static inline std::ostream& operator<<(std::ostream &os, rgw::sal::Lifecycle::LC
   return os;
 }
 
-static bool obj_has_expired(const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime, int days,
-                           ceph::real_time *expire_time = nullptr)
+static bool obj_has_expired(
+  const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime,
+  int days, ceph::real_time *expire_time = nullptr)
 {
   double timediff, cmp;
   utime_t base_time;
@@ -497,6 +505,37 @@ struct lc_op_ctx {
 }; /* lc_op_ctx */
 
 
+static bool pass_size_limit_checks(const DoutPrefixProvider *dpp, lc_op_ctx& oc) {
+
+  const auto& op = oc.op;
+  if (op.size_gt || op.size_lt) {
+    int ret{0};
+    auto& bucket = oc.bucket;
+    auto& o = oc.o;
+    std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(o.key);
+
+    RGWObjState *obj_state{nullptr};
+    ret = obj->get_obj_state(dpp, &obj_state, null_yield, true);
+    if (ret < 0) {
+      return false;
+    }
+
+    bool gt_p{true};
+    bool lt_p{true};
+
+    if (op.size_gt) {
+      gt_p = (obj_state->size > op.size_gt.get());
+    }
+    if (op.size_lt) {
+      lt_p = (obj_state->size < op.size_lt.get());
+    }
+
+    return gt_p && lt_p;
+  } /* require size check */
+
+  return true;
+}
+
 static std::string lc_id = "rgw lifecycle";
 static std::string lc_req_id = "0";
 
@@ -1122,10 +1161,14 @@ public:
       is_expired = obj_has_expired(dpp, oc.cct, mtime, op.expiration, exp_time);
     }
 
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+
     ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
-                     << (int)is_expired << " "
-                     << oc.wq->thr_name() << dendl;
-    return is_expired;
+                      << (int)is_expired << " size_check_p: "
+                      << size_check_p << " "
+                      << oc.wq->thr_name() << dendl;
+
+    return is_expired && size_check_p;
   }
 
   int process(lc_op_ctx& oc) override {
@@ -1183,14 +1226,18 @@ public:
     int expiration = oc.op.noncur_expiration;
     bool is_expired = obj_has_expired(dpp, oc.cct, oc.effective_mtime, expiration,
                                      exp_time);
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+    auto newer_noncurrent_p = (oc.num_noncurrent > oc.op.newer_noncurrent);
 
     ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
                       << is_expired << " " << ": num_noncurrent="
-                      << oc.num_noncurrent
+                      << oc.num_noncurrent << " size_check_p: "
+                      << size_check_p << " newer_noncurrent_p: "
+                      << newer_noncurrent_p << " "
                       << oc.wq->thr_name() << dendl;
 
     return is_expired &&
-      (oc.num_noncurrent > oc.op.newer_noncurrent) &&
+      (oc.num_noncurrent > oc.op.newer_noncurrent) && size_check_p &&
       pass_object_lock_check(oc.driver, oc.obj.get(), dpp);
   }
 
@@ -1299,15 +1346,18 @@ public:
       is_expired = obj_has_expired(dpp, oc.cct, mtime, transition.days, exp_time);
     }
 
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+
     ldpp_dout(oc.dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
-                     << is_expired << " "
-                     << oc.wq->thr_name() << dendl;
+                         << is_expired << " " << " size_check_p: "
+                         << size_check_p << " "
+                         << oc.wq->thr_name() << dendl;
 
     need_to_process =
       (rgw_placement_rule::get_canonical_storage_class(o.meta.storage_class) !=
        transition.storage_class);
 
-    return is_expired;
+    return is_expired && size_check_p;
   }
 
   bool should_process() override {

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index edfb661c56b427479deab058f59219be204a1b55..11a09c36be6bff86e3b01861d5e26da46941d5e6 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -206,6 +206,8 @@ class LCFilter
 
 protected:
   std::string prefix;
+  std::string size_gt;
+  std::string size_lt;
   RGWObjTags obj_tags;
   uint32_t flags;
 
@@ -227,13 +229,15 @@ public:
   }
 
   bool empty() const {
-    return !(has_prefix() || has_tags() || has_flags());
+    return !(has_prefix() || has_tags() || has_flags() ||
+            has_size_rule());
   }
 
   // Determine if we need AND tag when creating xml
   bool has_multi_condition() const {
-    if (obj_tags.count() + int(has_prefix()) + int(has_flags()) > 1) // Prefix is a member of Filter
-      return true;
+    if (obj_tags.count() + int(has_prefix()) + int(has_flags()) + int(has_size_rule()) > 1) {
+       return true;
+    }
     return false;
   }
 
@@ -245,6 +249,34 @@ public:
     return !obj_tags.empty();
   }
 
+  bool has_size_gt() const {
+    return !(size_gt.empty());
+  }
+
+  bool has_size_lt() const {
+    return !(size_lt.empty());
+  }
+
+  bool has_size_rule() const {
+    return (has_size_gt() || has_size_lt());
+  }
+
+  uint64_t get_size_gt() const {
+    uint64_t sz{0};
+    try {
+      sz = uint64_t(std::stoull(size_gt));
+    } catch (...) {}
+    return sz;
+  }
+
+  uint64_t get_size_lt() const {
+    uint64_t sz{0};
+    try {
+      sz = uint64_t(std::stoull(size_lt));
+    } catch (...) {}
+    return sz;
+  }
+
   bool has_flags() const {
     return !(flags == uint32_t(LCFlagType::none));
   }
@@ -254,10 +286,12 @@ public:
   }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(3, 1, bl);
+    ENCODE_START(4, 1, bl);
     encode(prefix, bl);
     encode(obj_tags, bl);
     encode(flags, bl);
+    encode(size_gt, bl);
+    encode(size_lt, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
@@ -267,6 +301,10 @@ public:
       decode(obj_tags, bl);
       if (struct_v >= 3) {
        decode(flags, bl);
+       if (struct_v >= 4) {
+         decode(size_gt, bl);
+         decode(size_lt, bl);
+       }
       }
     }
     DECODE_FINISH(bl);
@@ -450,8 +488,10 @@ struct lc_op
   bool dm_expiration{false};
   int expiration{0};
   int noncur_expiration{0};
-  int newer_noncurrent{0};
+  uint64_t newer_noncurrent{0};
   int mp_expiration{0};
+  boost::optional<uint64_t> size_gt;
+  boost::optional<uint64_t> size_lt;
   boost::optional<ceph::real_time> expiration_date;
   boost::optional<RGWObjTags> obj_tags;
   std::map<std::string, transition_action> transitions;

diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index e0b4337e509749e9d0eb1217f104fa8e4ff4dcc3..77d563b8be4a7af55458904be4a5e3d083eab0ba 100644 (file)
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -132,6 +132,12 @@ void LCFilter_S3::dump_xml(Formatter *f) const
       encode_xml("ArchiveZone", "", f);
     }
   }
+  if (has_size_gt()) {
+    encode_xml("ObjectSizeGreaterThanw", size_gt, f);
+  }
+  if (has_size_lt()) {
+    encode_xml("ObjectSizeLessThan", size_lt, f);
+  }
   if (multi) {
     f->close_section(); // And
   }
@@ -160,6 +166,13 @@ void LCFilter_S3::decode_xml(XMLObj *obj)
     flags |= make_flag(LCFlagType::ArchiveZone);
   }
 
+  RGWXMLDecoder::decode_xml("ObjectSizeGreaterThan", size_gt, o, false);
+  RGWXMLDecoder::decode_xml("ObjectSizeLessThan", size_lt, o, false);
+  if (has_size_gt() && has_size_lt() &&
+      (size_lt <= size_gt)) {
+    throw RGWXMLDecoder::err("Filter maximum object size must be larger than the minimum object size");
+  }
+
   obj_tags.clear(); // why is this needed?
   auto tags_iter = o->find("Tag");
   while (auto tag_xml = tags_iter.get_next()){
@@ -222,6 +235,13 @@ void LCRule_S3::decode_xml(XMLObj *obj)
 
   RGWXMLDecoder::decode_xml("ID", id, obj);
 
+  if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
+    throw RGWXMLDecoder::err("missing Status in Rule");
+  }
+  if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
+    throw RGWXMLDecoder::err("bad Status in Rule");
+  }
+
   LCFilter_S3 filter_s3;
   if (!RGWXMLDecoder::decode_xml("Filter", filter_s3, obj)) {
     // Ideally the following code should be deprecated and we should return
@@ -238,13 +258,6 @@ void LCRule_S3::decode_xml(XMLObj *obj)
   }
   filter = (LCFilter)filter_s3;
 
-  if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
-    throw RGWXMLDecoder::err("missing Status in Filter");
-  }
-  if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
-    throw RGWXMLDecoder::err("bad Status in Filter");
-  }
-
   LCExpiration_S3 s3_expiration;
   LCNoncurExpiration_S3 s3_noncur_expiration;
   LCMPExpiration_S3 s3_mp_expiration;

diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index f7a8c75d95aeed4e1aa15b31ed15e6153c7000aa..7d24db622938b60fe130d45a193940ff8a441db3 100644 (file)
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -149,6 +149,41 @@ TEST(TestLCConfigurationDecoder, XMLDoc4)
   ASSERT_EQ(noncur_expiration.get_newer(), 5);
 }
 
+static const char *xmldoc_5 =
+R"(<Rule>
+        <ID>expire-gt</ID>
+        <Expiration>
+            <Days>365</Days>
+        </Expiration>
+        <Filter>
+           <And>
+           <Prefix></Prefix>
+           <ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan>
+           <ObjectSizeLessThan>65536</ObjectSizeLessThan>
+           </And>
+        </Filter>
+        <Status>Enabled</Status>
+    </Rule>
+)";
+
+TEST(TestLCConfigurationDecoder, XMLDoc5)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  auto result1 = parser.parse(xmldoc_5, strlen(xmldoc_5), 1);
+  ASSERT_TRUE(result1);
+  LCRule_S3 rule;
+  auto result2 = RGWXMLDecoder::decode_xml("Rule", rule, &parser, true);
+  ASSERT_TRUE(result2);
+  /* check results */
+  ASSERT_TRUE(rule.is_enabled());
+  const auto& expiration = rule.get_expiration();
+  ASSERT_EQ(expiration.get_days(), 365);
+  const auto& filter = rule.get_filter();
+  ASSERT_EQ(filter.get_size_gt(), 1024);
+  ASSERT_EQ(filter.get_size_lt(), 65536);
+}
+
 struct LCWorkTimeTests : ::testing::Test
 {
    CephContext* cct;