]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
rgw: project and return lc expiration from GET/HEAD and PUT ops
authorMatt Benjamin <mbenjamin@redhat.com>
Sat, 26 Jan 2019 22:43:51 +0000 (17:43 -0500)
committerYuval Lifshitz <yuvalif@yahoo.com>
Tue, 3 Mar 2020 11:19:22 +0000 (13:19 +0200)
https://tracker.ceph.com/issues/38055

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit c89a889adc022f3e0298f6efd67a73a6a10a8bc7)
Signed-off-by: Yuval Lifshitz <yuvalif@yahoo.com>
Conflicts:
src/CMakeLists.txt
src/rgw/rgw_tag.h

src/rgw/rgw_common.h
src/rgw/rgw_lc.cc
src/rgw/rgw_lc.h
src/rgw/rgw_op.cc
src/rgw/rgw_rest_s3.cc
src/rgw/rgw_tag.cc
src/rgw/rgw_tag.h

index 4d0fe2541243d1d5cfaa42d18a8d4fcb5232117b..921a93add942a806e96741a9038c341b9077d008 100644 (file)
@@ -31,6 +31,7 @@
 #include "common/async/yield_context.h"
 #include "rgw_website.h"
 #include "rgw_object_lock.h"
+#include "rgw_tag.h"
 #include "cls/version/cls_version_types.h"
 #include "cls/user/cls_user_types.h"
 #include "cls/rgw/cls_rgw_types.h"
@@ -2082,6 +2083,8 @@ struct req_state : DoutPrefixProvider {
   string trans_id;
   uint64_t id;
 
+  RGWObjTags tagset;
+
   bool mfa_verified{false};
 
   /// optional coroutine context
index 951244396ac0872bc67041efbe8689da6ae6d03c..914d4fc7023d97982ad9f8d1ea8106d1bd14681e 100644 (file)
@@ -7,6 +7,7 @@
 
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
 
 #include "common/Formatter.h"
 #include <common/errno.h>
 #include "rgw_zone.h"
 #include "rgw_string.h"
 
+// this seems safe to use, at least for now--arguably, we should
+// prefer header-only fmt, in general
+#undef FMT_HEADER_ONLY
+#define FMT_HEADER_ONLY 1
+#include "seastar/fmt/include/fmt/format.h"
+
 #include "services/svc_sys_obj.h"
 
 #define dout_context g_ceph_context
@@ -1520,4 +1527,134 @@ int fix_lc_shard_entry(RGWRados* store, const RGWBucketInfo& bucket_info,
   return ret;
 }
 
-}
+std::string s3_expiration_header(
+  DoutPrefixProvider* dpp,
+  const rgw_obj_key& obj_key,
+  const RGWObjTags& obj_tagset,
+  const ceph::real_time& mtime,
+  const std::map<std::string, buffer::list>& bucket_attrs)
+{
+  CephContext* cct = dpp->get_cct();
+  RGWLifecycleConfiguration config(cct); // TODO: save in bucket info
+  std::string hdr{""};
+
+  const auto& aiter = bucket_attrs.find(RGW_ATTR_LC);
+  if (aiter == bucket_attrs.end())
+    return hdr;
+
+  bufferlist::const_iterator iter{&aiter->second};
+  try {
+      config.decode(iter);
+  } catch (const buffer::error& e) {
+      ldpp_dout(dpp, 0) << __func__
+                       <<  "() decode life cycle config failed"
+                       << dendl;
+      return hdr;
+  } /* catch */
+
+  boost::optional<ceph::real_time> expiration_date;
+  boost::optional<std::string> rule_id;
+
+  const auto& rule_map = config.get_rule_map();
+  for (const auto& ri : rule_map) {
+    const auto& rule = ri.second;
+    auto& id = rule.get_id();
+    auto& prefix = rule.get_prefix();
+    auto& filter = rule.get_filter();
+    auto& expiration = rule.get_expiration();
+    auto& noncur_expiration = rule.get_noncur_expiration();
+
+    ldpp_dout(dpp, 10) << "rule: " << ri.first
+                      << " prefix: " << prefix
+                      << " expiration: "
+                      << " date: " << expiration.get_date()
+                      << " days: " << expiration.get_days()
+                      << " noncur_expiration: "
+                      << " date: " << noncur_expiration.get_date()
+                      << " days: " << noncur_expiration.get_days()
+                      << dendl;
+
+    /* skip if rule !enabled
+     * if rule has prefix, skip iff object !match prefix
+     * if rule has tags, skip iff object !match tags
+     * note if object is current or non-current, compare accordingly
+     * if rule has days, construct date expression and save iff older
+     * than last saved
+     * if rule has date, convert date expression and save iff older
+     * than last saved
+     * if the date accum has a value, format it into hdr
+     */
+
+    if (!rule.is_enabled())
+      continue;
+
+    if(!prefix.empty()) {
+      if (!boost::starts_with(obj_key.name, prefix))
+       continue;
+    }
+
+    if (filter.has_tags()) {
+      bool tag_match = true;
+      const RGWObjTags& rule_tagset = filter.get_tags();
+      RGWObjTags::tag_map_t obj_tag_map = obj_tagset.get_tags();
+      for (auto& tag : rule_tagset.get_tags()) {
+       if (obj_tag_map.find(tag.first) == obj_tag_map.end()) {
+         tag_match = false;
+         break;
+       }
+      }
+      if (! tag_match)
+       continue;
+    }
+
+    // compute a uniform expiration date
+    boost::optional<ceph::real_time> rule_expiration_date;
+    const LCExpiration& rule_expiration =
+      (obj_key.instance.empty()) ? expiration : noncur_expiration;
+
+    if (rule_expiration.has_date()) {
+      rule_expiration_date =
+       boost::optional<ceph::real_time>(
+         ceph::from_iso_8601(rule.get_expiration().get_date()));
+      rule_id = id;
+    } else {
+      if (rule_expiration.has_days()) {
+       rule_expiration_date =
+         boost::optional<ceph::real_time>(
+           mtime + make_timespan(rule_expiration.get_days()*24*60*60));
+       rule_id = id;
+      }
+    }
+
+    // update earliest expiration
+    if (rule_expiration_date) {
+      if ((! expiration_date) ||
+         ((expiration_date &&
+           (*expiration_date < *rule_expiration_date)))) {
+      expiration_date =
+       boost::optional<ceph::real_time>(rule_expiration_date);
+      }
+    }
+  }
+
+  // cond format header
+  if (expiration_date && rule_id) {
+    // Fri, 23 Dec 2012 00:00:00 GMT
+    char exp_buf[100];
+    time_t exp = ceph::real_clock::to_time_t(*expiration_date);
+    if (std::strftime(exp_buf, sizeof(exp_buf),
+                     "%c", std::gmtime(&exp))) {
+      hdr = fmt::format("expiry-date=\"{0}\", rule-id=\"{1}\"", exp_buf,
+                       *rule_id);
+    } else {
+      ldpp_dout(dpp, 0) << __func__ <<
+       "() strftime of life cycle expiration header failed"
+                       << dendl;
+    }
+  }
+
+  return hdr;
+
+} /* rgwlc_s3_expiration_header */
+
+} /* namespace rgw::lc */
index df101e8370e07c8eef46efb8916aec4ca7947d8b..e50d9b75ce126860c62f9c6796b9d3ff6b32f510 100644 (file)
@@ -23,6 +23,7 @@
 #include "rgw_tag.h"
 
 #include <atomic>
+#include <tuple>
 
 #define HASH_PRIME 7877
 #define MAX_ID_LEN 255
@@ -211,8 +212,6 @@ class LCFilter
 };
 WRITE_CLASS_ENCODER(LCFilter)
 
-
-
 class LCRule
 {
 protected:
@@ -516,6 +515,13 @@ namespace rgw::lc {
 int fix_lc_shard_entry(RGWRados *store, const RGWBucketInfo& bucket_info,
                       const map<std::string,bufferlist>& battrs);
 
+std::string s3_expiration_header(
+  DoutPrefixProvider* dpp,
+  const rgw_obj_key& obj_key,
+  const RGWObjTags& obj_tagset,
+  const ceph::real_time& mtime,
+  const std::map<std::string, buffer::list>& bucket_attrs);
+
 } // namespace rgw::lc
 
 #endif
index 0005c9ee30dbf81c95133ccd8f6740eec938a711..239974d01050d08c55abcfa89e51bfb14f28ead3 100644 (file)
@@ -798,7 +798,7 @@ void rgw_add_to_iam_environment(rgw::IAM::Environment& e, std::string_view key,
 }
 
 static int rgw_iam_add_tags_from_bl(struct req_state* s, bufferlist& bl){
-  RGWObjTags tagset;
+  RGWObjTags& tagset = s->tagset;
   try {
     auto bliter = bl.cbegin();
     tagset.decode(bliter);
index 18b1f8fd92dae7eec6633d3f1f97f42b4a8ecfbf..37cf20af7864bfbaae72a537d3e27da50c09cde2 100644 (file)
@@ -95,6 +95,14 @@ void rgw_get_errno_s3(rgw_http_error *e , int err_no)
   }
 }
 
+static inline std::string get_s3_expiration_header(
+  struct req_state* s,
+  const ceph::real_time& mtime)
+{
+  return rgw::lc::s3_expiration_header(
+    s, s->object, s->tagset, mtime, s->bucket_attrs);
+}
+
 struct response_attr_param {
   const char *param;
   const char *http_attr;
@@ -187,6 +195,8 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
   map<string, string>::iterator riter;
   bufferlist metadata_bl;
 
+  string expires = get_s3_expiration_header(s, lastmod);
+
   if (sent_header)
     goto send_data;
 
@@ -258,6 +268,8 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
   dump_content_length(s, total_len);
   dump_last_modified(s, lastmod);
   dump_header_if_nonempty(s, "x-amz-version-id", version_id);
+  dump_header_if_nonempty(s, "x-amz-expiration", expires);
+
   if (attrs.find(RGW_ATTR_APPEND_PART_NUM) != attrs.end()) {
     dump_header(s, "x-rgw-object-type", "Appendable");
     dump_header(s, "x-rgw-next-append-position", s->obj_size);
@@ -1782,16 +1794,21 @@ void RGWPutObj_ObjStore_S3::send_response()
        s->cct->_conf->rgw_s3_success_create_obj_status);
       set_req_state_err(s, op_ret);
     }
+
+    string expires = get_s3_expiration_header(s, mtime);
+
     if (copy_source.empty()) {
       dump_errno(s);
       dump_etag(s, etag);
       dump_content_length(s, 0);
       dump_header_if_nonempty(s, "x-amz-version-id", version_id);
+      dump_header_if_nonempty(s, "x-amz-expiration", expires);
       for (auto &it : crypt_http_responses)
         dump_header(s, it.first, it.second);
     } else {
       dump_errno(s);
       dump_header_if_nonempty(s, "x-amz-version-id", version_id);
+      dump_header_if_nonempty(s, "x-amz-expiration", expires);
       end_header(s, this, "application/xml");
       dump_start(s);
       struct tm tmp;
index f793c0ab8b1a752c526a8ab71f263bfd6607f194..05c48bb1aeeee7d2ef7076eddaffe403d31f4f82 100644 (file)
@@ -8,6 +8,7 @@
 #include <boost/algorithm/string.hpp>
 
 #include "rgw_tag.h"
+#include "rgw_common.h"
 
 static constexpr uint32_t MAX_OBJ_TAGS=10;
 static constexpr uint32_t MAX_TAG_KEY_SIZE=128;
index fe8bbc348593a0722c53ff5cf74e2514fe3cfa62..80a18ae3cb33605fc6511ac98b03f4c56960b8ef 100644 (file)
@@ -8,16 +8,16 @@
 #include <include/types.h>
 #include <boost/container/flat_map.hpp>
 
-#include "rgw_common.h"
-
 class RGWObjTags
 {
- protected:
+public:
   using tag_map_t = boost::container::flat_map <std::string, std::string>;
+
+protected:
   tag_map_t tag_map;
  public:
-  RGWObjTags() {}
-  ~RGWObjTags() {}
+  RGWObjTags() = default;
+  ~RGWObjTags() = default;
 
   void encode(bufferlist& bl) const {
     ENCODE_START(1,1,bl);