From: Matt Benjamin Date: Fri, 19 Oct 2018 19:59:29 +0000 (-0400) Subject: rgw: handle S3 version 2 pre-signed urls with meta-data X-Git-Tag: v12.2.12~86^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=966c6dce171dfbe1998e66f87797db4f04454108;p=ceph.git rgw: handle S3 version 2 pre-signed urls with meta-data Add CanonicalizedAmzHeaders to the v2 signing document, as required in https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html Also lift qs metadata into x_meta_map in RGWPutObj_ObjStore_S3::get_parms(), so they are applied to the stored object. Fixes: https://tracker.ceph.com/issues/23470 Signed-off-by: Matt Benjamin (cherry picked from commit 3b0480ed136cebd3fba2322a77c7d047ef6ad5e3) Conflicts: src/rgw/rgw_rest_client.cc - Used 'method()' instead of 'method.c_str()' and 'new_url.c_str()' instead of 'url.c_str()' --- diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc index 2864f8a8cec..050503aaff4 100644 --- a/src/rgw/rgw_auth_s3.cc +++ b/src/rgw/rgw_auth_s3.cc @@ -116,6 +116,7 @@ void rgw_create_s3_canonical_header( const char* const content_type, const char* const date, const std::map& meta_map, + const std::map& qs_map, const char* const request_uri, const std::map& sub_resources, std::string& dest_str) @@ -143,6 +144,7 @@ void rgw_create_s3_canonical_header( dest.append("\n"); dest.append(get_canon_amz_hdr(meta_map)); + dest.append(get_canon_amz_hdr(qs_map)); dest.append(get_canon_resource(request_uri, sub_resources)); dest_str = dest; @@ -152,6 +154,17 @@ static inline bool is_base64_for_content_md5(unsigned char c) { return (isalnum(c) || isspace(c) || (c == '+') || (c == '/') || (c == '=')); } +static inline void get_v2_qs_map(const req_info& info, + std::map& qs_map) { + const auto& params = const_cast(info.args).get_params(); + for (const auto& elt : params) { + std::string k = boost::algorithm::to_lower_copy(elt.first); + if (k.find("x-amz-meta-") == /* offset */ 0) { + add_amz_meta_header(qs_map, k, elt.second); + } + } +} + /* * get the header authentication information required to * compute a request's signature @@ -175,7 +188,10 @@ bool rgw_create_s3_canonical_header(const req_info& info, const char *content_type = info.env->get("CONTENT_TYPE"); std::string date; + std::map qs_map; + if (qsr) { + get_v2_qs_map(info, qs_map); // handle qs metadata date = info.args.get("Expires"); } else { const char *str = info.env->get("HTTP_X_AMZ_DATE"); @@ -214,8 +230,8 @@ bool rgw_create_s3_canonical_header(const req_info& info, } rgw_create_s3_canonical_header(info.method, content_md5, content_type, - date.c_str(), meta_map, request_uri.c_str(), - sub_resources, dest); + date.c_str(), meta_map, qs_map, + request_uri.c_str(), sub_resources, dest); return true; } diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h index 5f6f70a7c66..1ad7dbc3f45 100644 --- a/src/rgw/rgw_auth_s3.h +++ b/src/rgw/rgw_auth_s3.h @@ -350,6 +350,7 @@ void rgw_create_s3_canonical_header( const char *content_type, const char *date, const std::map& meta_map, + const std::map& qs_map, const char *request_uri, const std::map& sub_resources, std::string& dest_str); diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 4980d233b39..5a2ed322142 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -6,8 +6,6 @@ #include #include #include -#include -#include #include "json_spirit/json_spirit.h" #include "common/ceph_json.h" @@ -404,7 +402,6 @@ struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"), STR_LEN_ENTRY("HTTP_X_ACCOUNT"), {NULL, 0} }; - void req_info::init_meta_info(bool *found_bad_meta) { x_meta_map.clear(); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 8e927ec9391..563fe7a79d1 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -18,6 +18,7 @@ #include +#include #include #include "common/ceph_crypto.h" @@ -2207,6 +2208,25 @@ static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes) return ((bytes + 4095) & ~4095) / 1024; } +/* implement combining step, S3 header canonicalization; k is a + * valid header and in lc form */ +static inline void add_amz_meta_header( + std::map& x_meta_map, + const std::string& k, + const std::string& v) +{ + auto it = x_meta_map.find(k); + if (it != x_meta_map.end()) { + std::string old = it->second; + boost::algorithm::trim_right(old); + old.append(","); + old.append(v); + x_meta_map[k] = old; + } else { + x_meta_map[k] = v; + } +} /* add_amz_meta_header */ + extern string rgw_string_unquote(const string& s); extern void parse_csv_string(const string& ival, vector& ovals); extern int parse_key_value(string& in_str, string& key, string& val); diff --git a/src/rgw/rgw_loadgen.cc b/src/rgw/rgw_loadgen.cc index fd66469e4d7..9b5e3401436 100644 --- a/src/rgw/rgw_loadgen.cc +++ b/src/rgw/rgw_loadgen.cc @@ -29,6 +29,7 @@ int RGWLoadGenRequestEnv::sign(RGWAccessKey& access_key) content_type.c_str(), date_str.c_str(), meta_map, + map{}, uri.c_str(), sub_resources, canonical_header); diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 22bbfe80f36..fada94f5764 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -123,7 +123,7 @@ int RGWRESTSimpleRequest::execute(RGWAccessKey& key, const char *method, const c map meta_map; map sub_resources; rgw_create_s3_canonical_header(method, NULL, NULL, date_str.c_str(), - meta_map, new_url.c_str(), sub_resources, + meta_map, meta_map, new_url.c_str(), sub_resources, canonical_header); string digest; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 7f7c7f0b0b2..27611e3dffb 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1243,6 +1243,19 @@ void RGWDeleteBucket_ObjStore_S3::send_response() } } +static inline void map_qs_metadata(struct req_state* s) +{ + /* merge S3 valid user metadata from the query-string into + * x_meta_map, which maps them to attributes */ + const auto& params = const_cast(s->info.args).get_params(); + for (const auto& elt : params) { + std::string k = boost::algorithm::to_lower_copy(elt.first); + if (k.find("x-amz-meta-") == /* offset */ 0) { + add_amz_meta_header(s->info.x_meta_map, k, elt.second); + } + } +} + int RGWPutObj_ObjStore_S3::get_params() { if (!s->length) @@ -1253,6 +1266,8 @@ int RGWPutObj_ObjStore_S3::get_params() size_t pos; int ret; + map_qs_metadata(s); + RGWAccessControlPolicy_S3 s3policy(s->cct); ret = create_s3_policy(s, store, s3policy, s->owner); if (ret < 0) @@ -1547,6 +1562,8 @@ int RGWPostObj_ObjStore_S3::get_params() return op_ret; } + map_qs_metadata(s); + ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name << dendl; env.add_var("bucket", s->bucket.name); @@ -2539,6 +2556,8 @@ int RGWCompleteMultipart_ObjStore_S3::get_params() return ret; } + map_qs_metadata(s); + return do_aws4_auth_completion(); }