From 2ac58ae264ecd9af5152cec8a736aaab9d7fefee Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Sat, 26 Jan 2019 17:43:51 -0500 Subject: [PATCH] rgw: project and return lc expiration from GET/HEAD and PUT ops https://tracker.ceph.com/issues/38055 Signed-off-by: Matt Benjamin (cherry picked from commit c89a889adc022f3e0298f6efd67a73a6a10a8bc7) Signed-off-by: Yuval Lifshitz Conflicts: src/CMakeLists.txt src/rgw/rgw_tag.h --- src/rgw/rgw_common.h | 3 + src/rgw/rgw_lc.cc | 139 ++++++++++++++++++++++++++++++++++++++++- src/rgw/rgw_lc.h | 10 ++- src/rgw/rgw_op.cc | 2 +- src/rgw/rgw_rest_s3.cc | 17 +++++ src/rgw/rgw_tag.cc | 1 + src/rgw/rgw_tag.h | 10 +-- 7 files changed, 173 insertions(+), 9 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 4d0fe2541243d..921a93add942a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -31,6 +31,7 @@ #include "common/async/yield_context.h" #include "rgw_website.h" #include "rgw_object_lock.h" +#include "rgw_tag.h" #include "cls/version/cls_version_types.h" #include "cls/user/cls_user_types.h" #include "cls/rgw/cls_rgw_types.h" @@ -2082,6 +2083,8 @@ struct req_state : DoutPrefixProvider { string trans_id; uint64_t id; + RGWObjTags tagset; + bool mfa_verified{false}; /// optional coroutine context diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 951244396ac08..914d4fc7023d9 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -7,6 +7,7 @@ #include #include +#include #include "common/Formatter.h" #include @@ -19,6 +20,12 @@ #include "rgw_zone.h" #include "rgw_string.h" +// this seems safe to use, at least for now--arguably, we should +// prefer header-only fmt, in general +#undef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY 1 +#include "seastar/fmt/include/fmt/format.h" + #include "services/svc_sys_obj.h" #define dout_context g_ceph_context @@ -1520,4 +1527,134 @@ int fix_lc_shard_entry(RGWRados* store, const RGWBucketInfo& bucket_info, return ret; } -} +std::string s3_expiration_header( + DoutPrefixProvider* dpp, + const rgw_obj_key& obj_key, + const RGWObjTags& obj_tagset, + const ceph::real_time& mtime, + const std::map& bucket_attrs) +{ + CephContext* cct = dpp->get_cct(); + RGWLifecycleConfiguration config(cct); // TODO: save in bucket info + std::string hdr{""}; + + const auto& aiter = bucket_attrs.find(RGW_ATTR_LC); + if (aiter == bucket_attrs.end()) + return hdr; + + bufferlist::const_iterator iter{&aiter->second}; + try { + config.decode(iter); + } catch (const buffer::error& e) { + ldpp_dout(dpp, 0) << __func__ + << "() decode life cycle config failed" + << dendl; + return hdr; + } /* catch */ + + boost::optional expiration_date; + boost::optional rule_id; + + const auto& rule_map = config.get_rule_map(); + for (const auto& ri : rule_map) { + const auto& rule = ri.second; + auto& id = rule.get_id(); + auto& prefix = rule.get_prefix(); + auto& filter = rule.get_filter(); + auto& expiration = rule.get_expiration(); + auto& noncur_expiration = rule.get_noncur_expiration(); + + ldpp_dout(dpp, 10) << "rule: " << ri.first + << " prefix: " << prefix + << " expiration: " + << " date: " << expiration.get_date() + << " days: " << expiration.get_days() + << " noncur_expiration: " + << " date: " << noncur_expiration.get_date() + << " days: " << noncur_expiration.get_days() + << dendl; + + /* skip if rule !enabled + * if rule has prefix, skip iff object !match prefix + * if rule has tags, skip iff object !match tags + * note if object is current or non-current, compare accordingly + * if rule has days, construct date expression and save iff older + * than last saved + * if rule has date, convert date expression and save iff older + * than last saved + * if the date accum has a value, format it into hdr + */ + + if (!rule.is_enabled()) + continue; + + if(!prefix.empty()) { + if (!boost::starts_with(obj_key.name, prefix)) + continue; + } + + if (filter.has_tags()) { + bool tag_match = true; + const RGWObjTags& rule_tagset = filter.get_tags(); + RGWObjTags::tag_map_t obj_tag_map = obj_tagset.get_tags(); + for (auto& tag : rule_tagset.get_tags()) { + if (obj_tag_map.find(tag.first) == obj_tag_map.end()) { + tag_match = false; + break; + } + } + if (! tag_match) + continue; + } + + // compute a uniform expiration date + boost::optional rule_expiration_date; + const LCExpiration& rule_expiration = + (obj_key.instance.empty()) ? expiration : noncur_expiration; + + if (rule_expiration.has_date()) { + rule_expiration_date = + boost::optional( + ceph::from_iso_8601(rule.get_expiration().get_date())); + rule_id = id; + } else { + if (rule_expiration.has_days()) { + rule_expiration_date = + boost::optional( + mtime + make_timespan(rule_expiration.get_days()*24*60*60)); + rule_id = id; + } + } + + // update earliest expiration + if (rule_expiration_date) { + if ((! expiration_date) || + ((expiration_date && + (*expiration_date < *rule_expiration_date)))) { + expiration_date = + boost::optional(rule_expiration_date); + } + } + } + + // cond format header + if (expiration_date && rule_id) { + // Fri, 23 Dec 2012 00:00:00 GMT + char exp_buf[100]; + time_t exp = ceph::real_clock::to_time_t(*expiration_date); + if (std::strftime(exp_buf, sizeof(exp_buf), + "%c", std::gmtime(&exp))) { + hdr = fmt::format("expiry-date=\"{0}\", rule-id=\"{1}\"", exp_buf, + *rule_id); + } else { + ldpp_dout(dpp, 0) << __func__ << + "() strftime of life cycle expiration header failed" + << dendl; + } + } + + return hdr; + +} /* rgwlc_s3_expiration_header */ + +} /* namespace rgw::lc */ diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index df101e8370e07..e50d9b75ce126 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -23,6 +23,7 @@ #include "rgw_tag.h" #include +#include #define HASH_PRIME 7877 #define MAX_ID_LEN 255 @@ -211,8 +212,6 @@ class LCFilter }; WRITE_CLASS_ENCODER(LCFilter) - - class LCRule { protected: @@ -516,6 +515,13 @@ namespace rgw::lc { int fix_lc_shard_entry(RGWRados *store, const RGWBucketInfo& bucket_info, const map& battrs); +std::string s3_expiration_header( + DoutPrefixProvider* dpp, + const rgw_obj_key& obj_key, + const RGWObjTags& obj_tagset, + const ceph::real_time& mtime, + const std::map& bucket_attrs); + } // namespace rgw::lc #endif diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 0005c9ee30dbf..239974d01050d 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -798,7 +798,7 @@ void rgw_add_to_iam_environment(rgw::IAM::Environment& e, std::string_view key, } static int rgw_iam_add_tags_from_bl(struct req_state* s, bufferlist& bl){ - RGWObjTags tagset; + RGWObjTags& tagset = s->tagset; try { auto bliter = bl.cbegin(); tagset.decode(bliter); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 18b1f8fd92dae..37cf20af7864b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -95,6 +95,14 @@ void rgw_get_errno_s3(rgw_http_error *e , int err_no) } } +static inline std::string get_s3_expiration_header( + struct req_state* s, + const ceph::real_time& mtime) +{ + return rgw::lc::s3_expiration_header( + s, s->object, s->tagset, mtime, s->bucket_attrs); +} + struct response_attr_param { const char *param; const char *http_attr; @@ -187,6 +195,8 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, map::iterator riter; bufferlist metadata_bl; + string expires = get_s3_expiration_header(s, lastmod); + if (sent_header) goto send_data; @@ -258,6 +268,8 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, dump_content_length(s, total_len); dump_last_modified(s, lastmod); dump_header_if_nonempty(s, "x-amz-version-id", version_id); + dump_header_if_nonempty(s, "x-amz-expiration", expires); + if (attrs.find(RGW_ATTR_APPEND_PART_NUM) != attrs.end()) { dump_header(s, "x-rgw-object-type", "Appendable"); dump_header(s, "x-rgw-next-append-position", s->obj_size); @@ -1782,16 +1794,21 @@ void RGWPutObj_ObjStore_S3::send_response() s->cct->_conf->rgw_s3_success_create_obj_status); set_req_state_err(s, op_ret); } + + string expires = get_s3_expiration_header(s, mtime); + if (copy_source.empty()) { dump_errno(s); dump_etag(s, etag); dump_content_length(s, 0); dump_header_if_nonempty(s, "x-amz-version-id", version_id); + dump_header_if_nonempty(s, "x-amz-expiration", expires); for (auto &it : crypt_http_responses) dump_header(s, it.first, it.second); } else { dump_errno(s); dump_header_if_nonempty(s, "x-amz-version-id", version_id); + dump_header_if_nonempty(s, "x-amz-expiration", expires); end_header(s, this, "application/xml"); dump_start(s); struct tm tmp; diff --git a/src/rgw/rgw_tag.cc b/src/rgw/rgw_tag.cc index f793c0ab8b1a7..05c48bb1aeeee 100644 --- a/src/rgw/rgw_tag.cc +++ b/src/rgw/rgw_tag.cc @@ -8,6 +8,7 @@ #include #include "rgw_tag.h" +#include "rgw_common.h" static constexpr uint32_t MAX_OBJ_TAGS=10; static constexpr uint32_t MAX_TAG_KEY_SIZE=128; diff --git a/src/rgw/rgw_tag.h b/src/rgw/rgw_tag.h index fe8bbc348593a..80a18ae3cb336 100644 --- a/src/rgw/rgw_tag.h +++ b/src/rgw/rgw_tag.h @@ -8,16 +8,16 @@ #include #include -#include "rgw_common.h" - class RGWObjTags { - protected: +public: using tag_map_t = boost::container::flat_map ; + +protected: tag_map_t tag_map; public: - RGWObjTags() {} - ~RGWObjTags() {} + RGWObjTags() = default; + ~RGWObjTags() = default; void encode(bufferlist& bl) const { ENCODE_START(1,1,bl); -- 2.39.5