From: Abhishek Lekshmanan Date: Fri, 3 Mar 2017 15:50:33 +0000 (+0100) Subject: rgw: make a configurable size for requests with xml params X-Git-Tag: v12.0.2~182^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=35375b75160dc93beeb04c7fe010dc0d21952534;p=ceph.git rgw: make a configurable size for requests with xml params We currently read all user input for a few apis accepting xml, avoid doing this and error out early, most s3 apis limit to about 1000 xml entries, make this configurable via a new config param, `rgw_max_put_param_size` defaulting to 1MB. Also modified `rgw_rest_read_all_input` with an additional param to not allow for chunked uploads, though we error out in a better way by responding with content length required rather than S3's 501 if chunked is set. This also adds the same behavior in RGWPutCORS reusing `rgw_rest_read_all_input`, and using a data_deleter to manage the char* Signed-off-by: Abhishek Lekshmanan rgw: rgw_rest_s3: make PUT CORS accept only a configured max input Signed-off-by: Abhishek Lekshmanan --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index a77a7788f78..509335bc8f1 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1402,6 +1402,7 @@ OPTION(rgw_max_chunk_size, OPT_INT, 4 * 1024 * 1024) OPTION(rgw_put_obj_min_window_size, OPT_INT, 16 * 1024 * 1024) OPTION(rgw_put_obj_max_window_size, OPT_INT, 64 * 1024 * 1024) OPTION(rgw_max_put_size, OPT_U64, 5ULL*1024*1024*1024) +OPTION(rgw_max_put_param_size, OPT_U64, 1 * 1024 * 1024) // max input size for PUT requests accepting json/xml params /** * override max bucket index shards in zone configuration (if not zero) diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index a6eff0e710d..c2e4dff8522 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1065,7 +1065,7 @@ public: class RGWPutACLs : public RGWOp { protected: - size_t len; + int len; char *data; ACLOwner owner; @@ -1108,7 +1108,7 @@ public: class RGWPutLC : public RGWOp { protected: - size_t len; + int len; char *data; string cookie; @@ -1524,7 +1524,7 @@ public: class RGWDeleteMultiObj : public RGWOp { protected: int max_to_delete; - size_t len; + int len; char *data; rgw_bucket bucket; bool quiet; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index c3c44fa4d04..1330911d426 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1253,55 +1253,19 @@ int RGWPostObj_ObjStore::verify_params() int RGWPutACLs_ObjStore::get_params() { - size_t cl = 0; - if (s->length) - cl = atoll(s->length); - if (cl) { - data = (char *)malloc(cl + 1); - if (!data) { - op_ret = -ENOMEM; - return op_ret; - } - const auto read_len = recv_body(s, data, cl); - if (read_len < 0) { - return read_len; - } else { - len = read_len; - } - data[len] = '\0'; - } else { - len = 0; - } - + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false); return op_ret; } int RGWPutLC_ObjStore::get_params() { - size_t cl = 0; - if (s->length) - cl = atoll(s->length); - if (cl) { - data = (char *)malloc(cl + 1); - if (!data) { - op_ret = -ENOMEM; - return op_ret; - } - const auto read_len = recv_body(s, data, cl); - if (read_len < 0) { - return read_len; - } else { - len = read_len; - } - data[len] = '\0'; - } else { - len = 0; - } - + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false); return op_ret; } -static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read) +static int read_all_chunked_input(req_state *s, char **pdata, int *plen, const uint64_t max_read) { #define READ_CHUNK 4096 #define MAX_READ_CHUNK (128 * 1024) @@ -1325,7 +1289,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max if (need_to_read < MAX_READ_CHUNK) need_to_read *= 2; - if (total > max_read) { + if ((unsigned)total > max_read) { free(data); return -ERANGE; } @@ -1351,7 +1315,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max } int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen, - int max_len) + const uint64_t max_len, const bool allow_chunked) { size_t cl = 0; int len = 0; @@ -1359,6 +1323,9 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen, if (s->length) cl = atoll(s->length); + else if (!allow_chunked) + return -ERR_LENGTH_REQUIRED; + if (cl) { if (cl > (size_t)max_len) { return -ERANGE; @@ -1373,7 +1340,7 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen, return len; } data[len] = '\0'; - } else if (!s->length) { + } else if (allow_chunked && !s->length) { const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING"); if (!encoding || strcmp(encoding, "chunked") != 0) return -ERR_LENGTH_REQUIRED; @@ -1461,28 +1428,8 @@ int RGWDeleteMultiObj_ObjStore::get_params() // everything is probably fine, set the bucket bucket = s->bucket; - size_t cl = 0; - - if (s->length) - cl = atoll(s->length); - if (cl) { - data = (char *)malloc(cl + 1); - if (!data) { - op_ret = -ENOMEM; - return op_ret; - } - const auto read_len = recv_body(s, data, cl); - if (read_len < 0) { - op_ret = read_len; - return op_ret; - } else { - len = read_len; - } - data[len] = '\0'; - } else { - return -EINVAL; - } - + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false); return op_ret; } diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 6e0a87ea4cb..765c0b93add 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -29,11 +29,11 @@ extern void rgw_flush_formatter(struct req_state *s, ceph::Formatter *formatter); extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen, - int max_len); + uint64_t max_len, bool allow_chunked=true); template int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, - int max_len, bool *empty) + uint64_t max_len, bool *empty) { int rv, data_len; char *data; @@ -72,7 +72,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, } template -int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len) +int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, uint64_t max_len, char **pdata, int *len) { int rv, data_len; char *data; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 277ff62c256..1b3ff56da4c 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -733,12 +733,10 @@ public: int RGWSetBucketVersioning_ObjStore_S3::get_params() { -#define GET_BUCKET_VERSIONING_BUF_MAX (128 * 1024) - char *data = nullptr; int len = 0; int r = - rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_VERSIONING_BUF_MAX); + rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false); if (r < 0) { return r; } @@ -786,11 +784,11 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response() int RGWSetBucketWebsite_ObjStore_S3::get_params() { - static constexpr uint32_t GET_BUCKET_WEBSITE_BUF_MAX = (128 * 1024); - char *data = nullptr; int len = 0; - int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX); + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + int r = rgw_rest_read_all_input(s, &data, &len, max_size, false); + if (r < 0) { return r; } @@ -959,8 +957,10 @@ int RGWCreateBucket_ObjStore_S3::get_params() int len = 0; char *data = nullptr; -#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */ - op_ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN); + + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false); + if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED)) return op_ret; @@ -2425,53 +2425,38 @@ void RGWGetCORS_ObjStore_S3::send_response() int RGWPutCORS_ObjStore_S3::get_params() { int r; - char *data = NULL; + char *data = nullptr; int len = 0; - size_t cl = 0; RGWCORSXMLParser_S3 parser(s->cct); RGWCORSConfiguration_S3 *cors_config; - if (s->length) - cl = atoll(s->length); - if (cl) { - data = (char *)malloc(cl + 1); - if (!data) { - r = -ENOMEM; - goto done_err; - } - len = recv_body(s, data, cl); - if (len < 0) { - r = len; - goto done_err; - } - data[len] = '\0'; - } else { - len = 0; + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + r = rgw_rest_read_all_input(s, &data, &len, max_size, false); + if (r < 0) { + return r; } + auto data_deleter = std::unique_ptr{data, free}; + if (s->aws4_auth_needs_complete) { - int ret_auth = do_aws4_auth_completion(); - if (ret_auth < 0) { - r = ret_auth; - goto done_err; + r = do_aws4_auth_completion(); + if (r < 0) { + return r; } } if (!parser.init()) { - r = -EINVAL; - goto done_err; + return -EINVAL; } if (!data || !parser.parse(data, len, 1)) { - r = -EINVAL; - goto done_err; + return -EINVAL; } cors_config = static_cast(parser.find_first( "CORSConfiguration")); if (!cors_config) { - r = -EINVAL; - goto done_err; + return -EINVAL; } if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) { @@ -2482,11 +2467,7 @@ int RGWPutCORS_ObjStore_S3::get_params() cors_config->encode(cors_bl); - free(data); return 0; -done_err: - free(data); - return r; } void RGWPutCORS_ObjStore_S3::send_response() @@ -2580,10 +2561,11 @@ public: int RGWSetRequestPayment_ObjStore_S3::get_params() { -#define GET_REQUEST_PAYMENT_BUF_MAX (128 * 1024) char *data; int len = 0; - int r = rgw_rest_read_all_input(s, &data, &len, GET_REQUEST_PAYMENT_BUF_MAX); + const auto max_size = s->cct->_conf->rgw_max_put_param_size; + int r = rgw_rest_read_all_input(s, &data, &len, max_size, false); + if (r < 0) { return r; }