From: Javier M. Mellid Date: Fri, 3 Jun 2016 15:34:10 +0000 (+0200) Subject: rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support X-Git-Tag: v10.2.3~42^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8cd7f44d8066a796cccd872dd2593582ac061331;p=ceph.git rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support When authenticating requests using the Authorization header in AWS4, you have the option of uploading the payload in chunks. You can send data in fixed size or variable size chunks. This patch enables streaming mode and signed headers support with chunked uploads. Fixes: http://tracker.ceph.com/issues/16146 Signed-off-by: Javier M. Mellid (cherry picked from commit 5de5876a535537f7878615898bb9cf7887204cb1) Conflicts: src/rgw/rgw_rest_s3.cc No change required to resolve the conflict. Manual merge was enough. --- diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc index aba0655ef7ab..8baa7992b185 100644 --- a/src/rgw/rgw_auth_s3.cc +++ b/src/rgw/rgw_auth_s3.cc @@ -277,7 +277,11 @@ void rgw_create_s3_v4_canonical_request(struct req_state *s, const string& canon if (s->aws4_auth_needs_complete) { request_payload_hash = STREAM_IO(s)->grab_aws4_sha256_hash(); } else { - rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash); + if (s->aws4_auth_streaming_mode) { + request_payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + } else { + rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash); + } } } @@ -392,13 +396,16 @@ int rgw_calculate_s3_v4_aws_signature(struct req_state *s, /* aws4_request */ - char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; + char *signing_k = s->aws4_auth->signing_k; + calc_hmac_sha256(service_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, "aws4_request", 12, signing_k); buf_to_hex((unsigned char *) signing_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, aux); ldout(s->cct, 10) << "signing_k = " << string(aux) << dendl; + s->aws4_auth->signing_key = aux; + /* new signature */ char signature_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index a76e2f200670..51a7eb315726 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -195,6 +195,7 @@ req_state::req_state(CephContext* _cct, RGWEnv* e, RGWUserInfo* u) object_acl = NULL; expect_cont = false; aws4_auth_needs_complete = false; + aws4_auth_streaming_mode = false; header_ended = false; obj_size = 0; diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index c94d19f29c61..7f40864a7d1d 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1188,6 +1188,10 @@ struct rgw_aws4_auth { string signature; string new_signature; string payload_hash; + string seed_signature; + string signing_key; + char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; + bufferlist bl; }; struct req_init_state { @@ -1260,6 +1264,7 @@ struct req_state { /* aws4 auth support */ bool aws4_auth_needs_complete; + bool aws4_auth_streaming_mode; unique_ptr aws4_auth; string canned_acl; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 218526b93ae2..5b19942d8712 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2422,8 +2422,8 @@ void RGWPutObj::execute() } do { - bufferlist data; - len = get_data(data); + bufferlist data_in; + len = get_data(data_in); if (len < 0) { op_ret = len; goto done; @@ -2431,6 +2431,15 @@ void RGWPutObj::execute() if (!len) break; + bufferlist data; + if (s->aws4_auth_streaming_mode) { + /* use unwrapped data */ + data = s->aws4_auth->bl; + len = data.length(); + } else { + data = data_in; + } + /* do we need this operation to be synchronous? if we're dealing with an object with immutable * head, e.g., multipart object we need to make sure we're the first one writing to this object */ @@ -2482,7 +2491,9 @@ void RGWPutObj::execute() ofs += len; } while (len > 0); - if (!chunked_upload && ofs != s->content_length) { + if (!chunked_upload && + ofs != s->content_length && + !s->aws4_auth_streaming_mode) { op_ret = -ERR_REQUEST_TIMEOUT; goto done; } @@ -2528,6 +2539,7 @@ void RGWPutObj::execute() hash.Final(m); buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); + etag = calc_md5; if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 826c7a9d78b9..0cdaca547686 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -997,6 +997,49 @@ int RGWPutObj_ObjStore::get_params() return 0; } +int RGWPutObj_ObjStore::get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size) { + + const int chunk_str_min_len = 1 + 17 + 64 + 2; /* len('0') = 1 */ + + char *chunk_str = bl.c_str(); + int budget = bl.length(); + + unsigned int chunk_data_size; + unsigned int chunk_offset = 0; + + while (1) { + + /* check available metadata */ + if (budget < chunk_str_min_len) { + return -ERR_SIGNATURE_NO_MATCH; + } + + chunk_offset = 0; + + /* grab chunk size */ + while ((*(chunk_str+chunk_offset) != ';') && (chunk_offset < chunk_str_min_len)) + chunk_offset++; + string str = string(chunk_str, chunk_offset); + stringstream ss; + ss << std::hex << str; + ss >> chunk_data_size; + + /* next chunk */ + chunk_offset += 17 + 64 + 2 + chunk_data_size; + + /* last chunk? */ + budget -= chunk_offset; + if (budget < 0) { + budget *= -1; + break; + } + + chunk_str += chunk_offset; + } + + return budget; +} + int RGWPutObj_ObjStore::get_data(bufferlist& bl) { size_t cl; @@ -1022,6 +1065,30 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl) len = read_len; bl.append(bp, 0, len); + + /* read last aws4 chunk padding */ + if (s->aws4_auth_streaming_mode && len == (int)chunk_size) { + int ret_auth = get_padding_last_aws4_chunk_encoded(bl, chunk_size); + if (ret_auth < 0) { + return ret_auth; + } + int len_padding = ret_auth; + if (len_padding) { + int read_len; + bufferptr bp_extra(len_padding); + int r = STREAM_IO(s)->read(bp_extra.c_str(), len_padding, &read_len, + s->aws4_auth_needs_complete); + if (r < 0) { + return r; + } + if (read_len != len_padding) { + return -ERR_SIGNATURE_NO_MATCH; + } + bl.append(bp_extra.c_str(), len_padding); + bl.rebuild(); + } + } + } if ((uint64_t)ofs + len > s->cct->_conf->rgw_max_put_size) { diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index b3568cf5794e..d7aa7c724037 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -217,6 +217,8 @@ public: virtual int verify_params(); virtual int get_params(); virtual int get_data(bufferlist& bl); + + int get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size); }; class RGWPostObj_ObjStore : public RGWPostObj diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index ec09449f613d..a8862e84680f 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1049,17 +1049,163 @@ int RGWPutObj_ObjStore_S3::get_params() return RGWPutObj_ObjStore::get_params(); } +int RGWPutObj_ObjStore_S3::validate_aws4_single_chunk(char *chunk_str, + char *chunk_data_str, + unsigned int chunk_data_size, + string chunk_signature) +{ + + /* string to sign */ + + string hash_empty_str; + rgw_hash_s3_string_sha256("", 0, hash_empty_str); + + string hash_chunk_data; + rgw_hash_s3_string_sha256(chunk_data_str, chunk_data_size, hash_chunk_data); + + string string_to_sign = "AWS4-HMAC-SHA256-PAYLOAD\n"; + string_to_sign.append(s->aws4_auth->date + "\n"); + string_to_sign.append(s->aws4_auth->credential_scope + "\n"); + string_to_sign.append(s->aws4_auth->seed_signature + "\n"); + string_to_sign.append(hash_empty_str + "\n"); + string_to_sign.append(hash_chunk_data); + + /* new chunk signature */ + + char signature_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; + calc_hmac_sha256(s->aws4_auth->signing_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, + string_to_sign.c_str(), string_to_sign.size(), signature_k); + + char aux[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE * 2 + 1]; + buf_to_hex((unsigned char *) signature_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, aux); + + string new_chunk_signature = string(aux); + + ldout(s->cct, 20) << "--------------- aws4 chunk validation" << dendl; + ldout(s->cct, 20) << "chunk_signature = " << chunk_signature << dendl; + ldout(s->cct, 20) << "new_chunk_signature = " << new_chunk_signature << dendl; + ldout(s->cct, 20) << "aws4 chunk signing_key = " << s->aws4_auth->signing_key << dendl; + ldout(s->cct, 20) << "aws4 chunk string_to_sign = " << string_to_sign << dendl; + + /* chunk auth ok? */ + + if (new_chunk_signature != chunk_signature) { + ldout(s->cct, 20) << "ERROR: AWS4 chunk signature does NOT match (new_chunk_signature != chunk_signature)" << dendl; + return -ERR_SIGNATURE_NO_MATCH; + } + + /* update seed signature */ + + s->aws4_auth->seed_signature = new_chunk_signature; + + return 0; +} + +int RGWPutObj_ObjStore_S3::validate_and_unwrap_available_aws4_chunked_data(bufferlist& bl_in, + bufferlist& bl_out) +{ + + /* string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n */ + + const unsigned int chunk_str_min_len = 1 + 17 + 64 + 2; /* len('0') = 1 */ + + char *chunk_str = bl_in.c_str(); + unsigned int budget = bl_in.length(); + + bl_out.clear(); + + while (true) { + + /* check available metadata */ + + if (budget < chunk_str_min_len) { + return -ERR_SIGNATURE_NO_MATCH; + } + + unsigned int chunk_offset = 0; + + /* grab chunk size */ + + while ((*(chunk_str+chunk_offset) != ';') && (chunk_offset < chunk_str_min_len)) + chunk_offset++; + string str = string(chunk_str, chunk_offset); + unsigned int chunk_data_size; + stringstream ss; + ss << std::hex << str; + ss >> chunk_data_size; + if (ss.fail()) { + return -ERR_SIGNATURE_NO_MATCH; + } + + /* grab chunk signature */ + + chunk_offset += 17; + string chunk_signature = string(chunk_str, chunk_offset, 64); + + /* get chunk data */ + + chunk_offset += 64 + 2; + char *chunk_data_str = chunk_str + chunk_offset; + + /* handle budget */ + + budget -= chunk_offset; + if (budget < chunk_data_size) { + return -ERR_SIGNATURE_NO_MATCH; + } else { + budget -= chunk_data_size; + } + + /* auth single chunk */ + + if (validate_aws4_single_chunk(chunk_str, chunk_data_str, chunk_data_size, chunk_signature) < 0) { + ldout(s->cct, 20) << "ERROR AWS4 single chunk validation" << dendl; + return -ERR_SIGNATURE_NO_MATCH; + } + + /* aggregate single chunk */ + + bl_out.append(chunk_data_str, chunk_data_size); + + /* last chunk or no more budget? */ + + if ((chunk_data_size == 0) || (budget == 0)) + break; + + /* next chunk */ + + chunk_offset += chunk_data_size; + chunk_str += chunk_offset; + } + + /* authorization ok */ + + return 0; + +} + int RGWPutObj_ObjStore_S3::get_data(bufferlist& bl) { int ret = RGWPutObj_ObjStore::get_data(bl); if (ret < 0) s->aws4_auth_needs_complete = false; + + int ret_auth; + + if (s->aws4_auth_streaming_mode && ret > 0) { + ret_auth = validate_and_unwrap_available_aws4_chunked_data(bl, s->aws4_auth->bl); + if (ret_auth < 0) { + return ret_auth; + } + } + if ((ret == 0) && s->aws4_auth_needs_complete) { - int ret_auth = do_aws4_auth_completion(); + ret_auth = do_aws4_auth_completion(); if (ret_auth < 0) { return ret_auth; } } + return ret; } @@ -3117,7 +3263,7 @@ int RGW_Auth_S3::authorize_v4_complete(RGWRados *store, struct req_state *s, con if (s->aws4_auth_needs_complete) { const char *expected_request_payload_hash = s->info.env->get("HTTP_X_AMZ_CONTENT_SHA256"); if (expected_request_payload_hash && - s->aws4_auth->payload_hash.compare(expected_request_payload_hash) != 0) { + s->aws4_auth->payload_hash.compare(expected_request_payload_hash) != 0) { ldout(s->cct, 10) << "ERROR: x-amz-content-sha256 does not match" << dendl; return -ERR_AMZ_CONTENT_SHA256_MISMATCH; } @@ -3168,6 +3314,8 @@ int RGW_Auth_S3::authorize_v4_complete(RGWRados *store, struct req_state *s, con return err; } + s->aws4_auth->seed_signature = s->aws4_auth->new_signature; + return 0; } @@ -3515,6 +3663,7 @@ int RGW_Auth_S3::authorize_v4(RGWRados *store, struct req_state *s) string request_payload; bool unsigned_payload = false; + s->aws4_auth_streaming_mode = false; if (using_qs) { /* query parameters auth */ @@ -3522,8 +3671,11 @@ int RGW_Auth_S3::authorize_v4(RGWRados *store, struct req_state *s) } else { /* header auth */ const char *request_payload_hash = s->info.env->get("HTTP_X_AMZ_CONTENT_SHA256"); - if (request_payload_hash && string("UNSIGNED-PAYLOAD").compare(request_payload_hash) == 0) { - unsigned_payload = true; + if (request_payload_hash) { + unsigned_payload = string("UNSIGNED-PAYLOAD").compare(request_payload_hash) == 0; + if (!unsigned_payload) { + s->aws4_auth_streaming_mode = string("STREAMING-AWS4-HMAC-SHA256-PAYLOAD").compare(request_payload_hash) == 0; + } } } @@ -3563,26 +3715,65 @@ int RGW_Auth_S3::authorize_v4(RGWRados *store, struct req_state *s) /* aws4 auth not completed... delay aws4 auth */ - dout(10) << "body content detected... delaying v4 auth" << dendl; - - switch (s->op_type) - { - case RGW_OP_CREATE_BUCKET: - case RGW_OP_PUT_OBJ: - case RGW_OP_PUT_ACLS: - case RGW_OP_PUT_CORS: - case RGW_OP_COMPLETE_MULTIPART: - case RGW_OP_SET_BUCKET_VERSIONING: - case RGW_OP_DELETE_MULTI_OBJ: - case RGW_OP_ADMIN_SET_METADATA: - case RGW_OP_SET_BUCKET_WEBSITE: - break; - default: - dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl; - return -ERR_NOT_IMPLEMENTED; - } + if (!s->aws4_auth_streaming_mode) { + + dout(10) << "delaying v4 auth" << dendl; + + /* payload in a single chunk */ + + switch (s->op_type) + { + case RGW_OP_CREATE_BUCKET: + case RGW_OP_PUT_OBJ: + case RGW_OP_PUT_ACLS: + case RGW_OP_PUT_CORS: + case RGW_OP_COMPLETE_MULTIPART: + case RGW_OP_SET_BUCKET_VERSIONING: + case RGW_OP_DELETE_MULTI_OBJ: + case RGW_OP_ADMIN_SET_METADATA: + case RGW_OP_SET_BUCKET_WEBSITE: + break; + default: + dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl; + return -ERR_NOT_IMPLEMENTED; + } + + s->aws4_auth_needs_complete = true; + + } else { + + dout(10) << "body content detected in multiple chunks" << dendl; + + /* payload in multiple chunks */ - s->aws4_auth_needs_complete = true; + switch(s->op_type) + { + case RGW_OP_PUT_OBJ: + break; + default: + dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl; + return -ERR_NOT_IMPLEMENTED; + } + + /* calculate seed */ + + int err = authorize_v4_complete(store, s, "", unsigned_payload); + if (err) { + return err; + } + + /* verify seed signature */ + + if (s->aws4_auth->signature != s->aws4_auth->new_signature) { + dout(10) << "ERROR: AWS4 seed signature does NOT match!" << dendl; + return -ERR_SIGNATURE_NO_MATCH; + } + + dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl; + + s->aws4_auth_needs_complete = false; + + } } diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 1482a95287f0..4d432247c698 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -162,6 +162,13 @@ public: int get_params(); int get_data(bufferlist& bl); void send_response(); + + int validate_aws4_single_chunk(char *chunk_str, + char *chunk_data_str, + unsigned int chunk_data_size, + string chunk_signature); + int validate_and_unwrap_available_aws4_chunked_data(bufferlist& bl_in, + bufferlist& bl_out); }; struct post_part_field {