From: Matt Benjamin Date: Tue, 24 Oct 2023 21:59:34 +0000 (-0400) Subject: rgwlc: implement Filter size limit checks X-Git-Tag: testing/wip-root-testing-20240411.174241~121^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=789b2ffeba1dd838f546cce013859f8024916379;p=ceph-ci.git rgwlc: implement Filter size limit checks i.e., this implements ObjectSizeGreaterThan and ObjectSizeLessThan for supported expiration and transition actions. Fixes: https://tracker.ceph.com/issues/63304 Signed-off-by: Matt Benjamin (cherry picked from commit c9b19be15362c63526e92f4d968f942a40cab150) --- diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index feee38d1f75..7169e09edbc 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -155,10 +155,17 @@ bool RGWLifecycleConfiguration::_add_rule(const LCRule& rule) } else { prefix = rule.get_prefix(); } - if (rule.get_filter().has_tags()){ - op.obj_tags = rule.get_filter().get_tags(); + const auto& filter = rule.get_filter(); + if (filter.has_tags()){ + op.obj_tags = filter.get_tags(); } - op.rule_flags = rule.get_filter().get_flags(); + if (filter.has_size_gt()) { + op.size_gt = filter.get_size_gt(); + } + if (filter.has_size_lt()) { + op.size_lt = filter.get_size_lt(); + } + op.rule_flags = filter.get_flags(); prefix_map.emplace(std::move(prefix), std::move(op)); return true; } @@ -262,8 +269,9 @@ static inline std::ostream& operator<<(std::ostream &os, rgw::sal::Lifecycle::LC return os; } -static bool obj_has_expired(const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime, int days, - ceph::real_time *expire_time = nullptr) +static bool obj_has_expired( + const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime, + int days, ceph::real_time *expire_time = nullptr) { double timediff, cmp; utime_t base_time; @@ -497,6 +505,37 @@ struct lc_op_ctx { }; /* lc_op_ctx */ +static bool pass_size_limit_checks(const DoutPrefixProvider *dpp, lc_op_ctx& oc) { + + const auto& op = oc.op; + if (op.size_gt || op.size_lt) { + int ret{0}; + auto& bucket = oc.bucket; + auto& o = oc.o; + std::unique_ptr obj = bucket->get_object(o.key); + + RGWObjState *obj_state{nullptr}; + ret = obj->get_obj_state(dpp, &obj_state, null_yield, true); + if (ret < 0) { + return false; + } + + bool gt_p{true}; + bool lt_p{true}; + + if (op.size_gt) { + gt_p = (obj_state->size > op.size_gt.get()); + } + if (op.size_lt) { + lt_p = (obj_state->size < op.size_lt.get()); + } + + return gt_p && lt_p; + } /* require size check */ + + return true; +} + static std::string lc_id = "rgw lifecycle"; static std::string lc_req_id = "0"; @@ -1122,10 +1161,14 @@ public: is_expired = obj_has_expired(dpp, oc.cct, mtime, op.expiration, exp_time); } + auto size_check_p = pass_size_limit_checks(dpp, oc); + ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired=" - << (int)is_expired << " " - << oc.wq->thr_name() << dendl; - return is_expired; + << (int)is_expired << " size_check_p: " + << size_check_p << " " + << oc.wq->thr_name() << dendl; + + return is_expired && size_check_p; } int process(lc_op_ctx& oc) override { @@ -1183,14 +1226,18 @@ public: int expiration = oc.op.noncur_expiration; bool is_expired = obj_has_expired(dpp, oc.cct, oc.effective_mtime, expiration, exp_time); + auto size_check_p = pass_size_limit_checks(dpp, oc); + auto newer_noncurrent_p = (oc.num_noncurrent > oc.op.newer_noncurrent); ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired=" << is_expired << " " << ": num_noncurrent=" - << oc.num_noncurrent + << oc.num_noncurrent << " size_check_p: " + << size_check_p << " newer_noncurrent_p: " + << newer_noncurrent_p << " " << oc.wq->thr_name() << dendl; return is_expired && - (oc.num_noncurrent > oc.op.newer_noncurrent) && + (oc.num_noncurrent > oc.op.newer_noncurrent) && size_check_p && pass_object_lock_check(oc.driver, oc.obj.get(), dpp); } @@ -1299,15 +1346,18 @@ public: is_expired = obj_has_expired(dpp, oc.cct, mtime, transition.days, exp_time); } + auto size_check_p = pass_size_limit_checks(dpp, oc); + ldpp_dout(oc.dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired=" - << is_expired << " " - << oc.wq->thr_name() << dendl; + << is_expired << " " << " size_check_p: " + << size_check_p << " " + << oc.wq->thr_name() << dendl; need_to_process = (rgw_placement_rule::get_canonical_storage_class(o.meta.storage_class) != transition.storage_class); - return is_expired; + return is_expired && size_check_p; } bool should_process() override { diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index edfb661c56b..11a09c36be6 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -206,6 +206,8 @@ class LCFilter protected: std::string prefix; + std::string size_gt; + std::string size_lt; RGWObjTags obj_tags; uint32_t flags; @@ -227,13 +229,15 @@ public: } bool empty() const { - return !(has_prefix() || has_tags() || has_flags()); + return !(has_prefix() || has_tags() || has_flags() || + has_size_rule()); } // Determine if we need AND tag when creating xml bool has_multi_condition() const { - if (obj_tags.count() + int(has_prefix()) + int(has_flags()) > 1) // Prefix is a member of Filter - return true; + if (obj_tags.count() + int(has_prefix()) + int(has_flags()) + int(has_size_rule()) > 1) { + return true; + } return false; } @@ -245,6 +249,34 @@ public: return !obj_tags.empty(); } + bool has_size_gt() const { + return !(size_gt.empty()); + } + + bool has_size_lt() const { + return !(size_lt.empty()); + } + + bool has_size_rule() const { + return (has_size_gt() || has_size_lt()); + } + + uint64_t get_size_gt() const { + uint64_t sz{0}; + try { + sz = uint64_t(std::stoull(size_gt)); + } catch (...) {} + return sz; + } + + uint64_t get_size_lt() const { + uint64_t sz{0}; + try { + sz = uint64_t(std::stoull(size_lt)); + } catch (...) {} + return sz; + } + bool has_flags() const { return !(flags == uint32_t(LCFlagType::none)); } @@ -254,10 +286,12 @@ public: } void encode(bufferlist& bl) const { - ENCODE_START(3, 1, bl); + ENCODE_START(4, 1, bl); encode(prefix, bl); encode(obj_tags, bl); encode(flags, bl); + encode(size_gt, bl); + encode(size_lt, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { @@ -267,6 +301,10 @@ public: decode(obj_tags, bl); if (struct_v >= 3) { decode(flags, bl); + if (struct_v >= 4) { + decode(size_gt, bl); + decode(size_lt, bl); + } } } DECODE_FINISH(bl); @@ -450,8 +488,10 @@ struct lc_op bool dm_expiration{false}; int expiration{0}; int noncur_expiration{0}; - int newer_noncurrent{0}; + uint64_t newer_noncurrent{0}; int mp_expiration{0}; + boost::optional size_gt; + boost::optional size_lt; boost::optional expiration_date; boost::optional obj_tags; std::map transitions; diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc index e0b4337e509..77d563b8be4 100644 --- a/src/rgw/rgw_lc_s3.cc +++ b/src/rgw/rgw_lc_s3.cc @@ -132,6 +132,12 @@ void LCFilter_S3::dump_xml(Formatter *f) const encode_xml("ArchiveZone", "", f); } } + if (has_size_gt()) { + encode_xml("ObjectSizeGreaterThanw", size_gt, f); + } + if (has_size_lt()) { + encode_xml("ObjectSizeLessThan", size_lt, f); + } if (multi) { f->close_section(); // And } @@ -160,6 +166,13 @@ void LCFilter_S3::decode_xml(XMLObj *obj) flags |= make_flag(LCFlagType::ArchiveZone); } + RGWXMLDecoder::decode_xml("ObjectSizeGreaterThan", size_gt, o, false); + RGWXMLDecoder::decode_xml("ObjectSizeLessThan", size_lt, o, false); + if (has_size_gt() && has_size_lt() && + (size_lt <= size_gt)) { + throw RGWXMLDecoder::err("Filter maximum object size must be larger than the minimum object size"); + } + obj_tags.clear(); // why is this needed? auto tags_iter = o->find("Tag"); while (auto tag_xml = tags_iter.get_next()){ @@ -222,6 +235,13 @@ void LCRule_S3::decode_xml(XMLObj *obj) RGWXMLDecoder::decode_xml("ID", id, obj); + if (!RGWXMLDecoder::decode_xml("Status", status, obj)) { + throw RGWXMLDecoder::err("missing Status in Rule"); + } + if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) { + throw RGWXMLDecoder::err("bad Status in Rule"); + } + LCFilter_S3 filter_s3; if (!RGWXMLDecoder::decode_xml("Filter", filter_s3, obj)) { // Ideally the following code should be deprecated and we should return @@ -238,13 +258,6 @@ void LCRule_S3::decode_xml(XMLObj *obj) } filter = (LCFilter)filter_s3; - if (!RGWXMLDecoder::decode_xml("Status", status, obj)) { - throw RGWXMLDecoder::err("missing Status in Filter"); - } - if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) { - throw RGWXMLDecoder::err("bad Status in Filter"); - } - LCExpiration_S3 s3_expiration; LCNoncurExpiration_S3 s3_noncur_expiration; LCMPExpiration_S3 s3_mp_expiration; diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc index f7a8c75d95a..7d24db62293 100644 --- a/src/test/rgw/test_rgw_lc.cc +++ b/src/test/rgw/test_rgw_lc.cc @@ -149,6 +149,41 @@ TEST(TestLCConfigurationDecoder, XMLDoc4) ASSERT_EQ(noncur_expiration.get_newer(), 5); } +static const char *xmldoc_5 = +R"( + expire-gt + + 365 + + + + + 1024 + 65536 + + + Enabled + +)"; + +TEST(TestLCConfigurationDecoder, XMLDoc5) +{ + RGWXMLDecoder::XMLParser parser; + ASSERT_TRUE(parser.init()); + auto result1 = parser.parse(xmldoc_5, strlen(xmldoc_5), 1); + ASSERT_TRUE(result1); + LCRule_S3 rule; + auto result2 = RGWXMLDecoder::decode_xml("Rule", rule, &parser, true); + ASSERT_TRUE(result2); + /* check results */ + ASSERT_TRUE(rule.is_enabled()); + const auto& expiration = rule.get_expiration(); + ASSERT_EQ(expiration.get_days(), 365); + const auto& filter = rule.get_filter(); + ASSERT_EQ(filter.get_size_gt(), 1024); + ASSERT_EQ(filter.get_size_lt(), 65536); +} + struct LCWorkTimeTests : ::testing::Test { CephContext* cct;