rgw: handle S3 version 2 pre-signed urls with meta-data

Add CanonicalizedAmzHeaders to the v2 signing document, as required
in https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html

Also lift qs metadata into x_meta_map in RGWPutObj_ObjStore_S3::get_parms(),
so they are applied to the stored object.

Fixes: https://tracker.ceph.com/issues/23470
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit 3b0480ed136cebd3fba2322a77c7d047ef6ad5e3)

diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 6534bfb440677c09ebabcbf6832d332413626eb8..397ec872794b89b71413828b9f007be3b6d31b06 100644 (file)
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -116,6 +116,7 @@ void rgw_create_s3_canonical_header(
   const char* const content_type,
   const char* const date,
   const std::map<std::string, std::string>& meta_map,
+  const std::map<std::string, std::string>& qs_map,
   const char* const request_uri,
   const std::map<std::string, std::string>& sub_resources,
   std::string& dest_str)
@@ -143,6 +144,7 @@ void rgw_create_s3_canonical_header(
   dest.append("\n");
 
   dest.append(get_canon_amz_hdr(meta_map));
+  dest.append(get_canon_amz_hdr(qs_map));
   dest.append(get_canon_resource(request_uri, sub_resources));
 
   dest_str = dest;
@@ -152,6 +154,17 @@ static inline bool is_base64_for_content_md5(unsigned char c) {
   return (isalnum(c) || isspace(c) || (c == '+') || (c == '/') || (c == '='));
 }
 
+static inline void get_v2_qs_map(const req_info& info,
+                                std::map<std::string, std::string>& qs_map) {
+  const auto& params = const_cast<RGWHTTPArgs&>(info.args).get_params();
+  for (const auto& elt : params) {
+    std::string k = boost::algorithm::to_lower_copy(elt.first);
+    if (k.find("x-amz-meta-") == /* offset */ 0) {
+      add_amz_meta_header(qs_map, k, elt.second);
+    }
+  }
+}
+
 /*
  * get the header authentication  information required to
  * compute a request's signature
@@ -175,7 +188,10 @@ bool rgw_create_s3_canonical_header(const req_info& info,
   const char *content_type = info.env->get("CONTENT_TYPE");
 
   std::string date;
+  std::map<std::string, std::string> qs_map;
+
   if (qsr) {
+    get_v2_qs_map(info, qs_map); // handle qs metadata
     date = info.args.get("Expires");
   } else {
     const char *str = info.env->get("HTTP_X_AMZ_DATE");
@@ -214,8 +230,8 @@ bool rgw_create_s3_canonical_header(const req_info& info,
   }
 
   rgw_create_s3_canonical_header(info.method, content_md5, content_type,
-                                 date.c_str(), meta_map, request_uri.c_str(),
-                                 sub_resources, dest);
+                                 date.c_str(), meta_map, qs_map,
+                                request_uri.c_str(), sub_resources, dest);
   return true;
 }
 

diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h
index 2934484167a4ea1f8fd1e0a3830935a017517c02..37d808b4ce36eb543a0e50cb57ea0cb5366655ab 100644 (file)
--- a/src/rgw/rgw_auth_s3.h
+++ b/src/rgw/rgw_auth_s3.h
@@ -345,6 +345,7 @@ void rgw_create_s3_canonical_header(
   const char *content_type,
   const char *date,
   const std::map<std::string, std::string>& meta_map,
+  const std::map<std::string, std::string>& qs_map,
   const char *request_uri,
   const std::map<std::string, std::string>& sub_resources,
   std::string& dest_str);

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 34004b047c21579c538b6f16841f6a9a35df8ffd..9c2d1e159f6b49e8a899a9adf750695e701d64b1 100644 (file)
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -6,8 +6,6 @@
 #include <algorithm>
 #include <string>
 #include <boost/tokenizer.hpp>
-#include <boost/algorithm/string.hpp>
-#include <boost/utility/string_view.hpp>
 
 #include "json_spirit/json_spirit.h"
 #include "common/ceph_json.h"
@@ -384,7 +382,6 @@ struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"),
                                    STR_LEN_ENTRY("HTTP_X_ACCOUNT"),
                                    {NULL, 0} };
 
-
 void req_info::init_meta_info(bool *found_bad_meta)
 {
   x_meta_map.clear();

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 9a8e6c6021ca8c96cf2f10d675fe162eed0491c4..cf00f29704b8914759f71c1ce2df83c9a202504f 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -18,6 +18,7 @@
 
 #include <array>
 
+#include <boost/algorithm/string.hpp>
 #include <boost/utility/string_view.hpp>
 
 #include "common/ceph_crypto.h"
@@ -2243,6 +2244,25 @@ static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
   return ((bytes + 4095) & ~4095) / 1024;
 }
 
+/* implement combining step, S3 header canonicalization;  k is a
+ * valid header and in lc form */
+static inline void add_amz_meta_header(
+  std::map<std::string, std::string>& x_meta_map,
+  const std::string& k,
+  const std::string& v)
+{
+  auto it = x_meta_map.find(k);
+  if (it != x_meta_map.end()) {
+    std::string old = it->second;
+    boost::algorithm::trim_right(old);
+    old.append(",");
+    old.append(v);
+    x_meta_map[k] = old;
+  } else {
+    x_meta_map[k] = v;
+  }
+} /* add_amz_meta_header */
+
 extern string rgw_string_unquote(const string& s);
 extern void parse_csv_string(const string& ival, vector<string>& ovals);
 extern int parse_key_value(string& in_str, string& key, string& val);

diff --git a/src/rgw/rgw_loadgen.cc b/src/rgw/rgw_loadgen.cc
index fd66469e4d7d99116879c8b77b1e3e56df927de5..9b5e3401436072bfe92f6746c7fffdc5f9ea4df1 100644 (file)
--- a/src/rgw/rgw_loadgen.cc
+++ b/src/rgw/rgw_loadgen.cc
@@ -29,6 +29,7 @@ int RGWLoadGenRequestEnv::sign(RGWAccessKey& access_key)
                                  content_type.c_str(),
                                  date_str.c_str(),
                                  meta_map,
+                                map<string, string>{},
                                  uri.c_str(),
                                  sub_resources,
                                  canonical_header);

diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index cd8b8e08122e154cddfeaabdae51947b0d2fa069..dda936085885eb1e18389895c05dac2868fd71e4 100644 (file)
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -140,9 +140,10 @@ int RGWRESTSimpleRequest::execute(RGWAccessKey& key, const char *_method, const
   string canonical_header;
   map<string, string> meta_map;
   map<string, string> sub_resources;
+
   rgw_create_s3_canonical_header(method.c_str(), NULL, NULL, date_str.c_str(),
-                            meta_map, url.c_str(), sub_resources,
-                            canonical_header);
+                                meta_map, meta_map, url.c_str(), sub_resources,
+                                canonical_header);
 
   string digest;
   try {

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index add3eda61d29e4bcda6191ed2264075b7d37e358..685ca9126a4b20b73123e01e9cbfd67e1f51999b 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1267,6 +1267,19 @@ void RGWDeleteBucket_ObjStore_S3::send_response()
   }
 }
 
+static inline void map_qs_metadata(struct req_state* s)
+{
+  /* merge S3 valid user metadata from the query-string into
+   * x_meta_map, which maps them to attributes */
+  const auto& params = const_cast<RGWHTTPArgs&>(s->info.args).get_params();
+  for (const auto& elt : params) {
+    std::string k = boost::algorithm::to_lower_copy(elt.first);
+    if (k.find("x-amz-meta-") == /* offset */ 0) {
+      add_amz_meta_header(s->info.x_meta_map, k, elt.second);
+    }
+  }
+}
+
 int RGWPutObj_ObjStore_S3::get_params()
 {
   if (!s->length)
@@ -1277,6 +1290,8 @@ int RGWPutObj_ObjStore_S3::get_params()
   size_t pos;
   int ret;
 
+  map_qs_metadata(s);
+
   RGWAccessControlPolicy_S3 s3policy(s->cct);
   ret = create_s3_policy(s, store, s3policy, s->owner);
   if (ret < 0)
@@ -1573,6 +1588,8 @@ int RGWPostObj_ObjStore_S3::get_params()
     return op_ret;
   }
 
+  map_qs_metadata(s);
+
   ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name
                    << dendl;
   env.add_var("bucket", s->bucket.name);
@@ -2566,6 +2583,8 @@ int RGWCompleteMultipart_ObjStore_S3::get_params()
     return ret;
   }
 
+  map_qs_metadata(s);
+
   return do_aws4_auth_completion();
 }