rgw: make a configurable size for requests with xml params

We currently read all user input for a few apis accepting xml, avoid
doing this and error out early, most s3 apis limit to about 1000 xml
entries, make this configurable via a new config param,
`rgw_max_put_param_size` defaulting to 1MB. Also modified
`rgw_rest_read_all_input` with an additional param to not allow for
chunked uploads, though we error out in a better way by responding with
content length required rather than S3's 501 if chunked is set.

This also adds the same behavior in RGWPutCORS reusing
`rgw_rest_read_all_input`, and using a data_deleter to manage the char*

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
rgw: rgw_rest_s3: make PUT CORS accept only a configured max input

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index a77a7788f78814fd850787d8015ec075953fba3c..509335bc8f1725c8aae95762638ea378d4176d23 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1402,6 +1402,7 @@ OPTION(rgw_max_chunk_size, OPT_INT, 4 * 1024 * 1024)
 OPTION(rgw_put_obj_min_window_size, OPT_INT, 16 * 1024 * 1024)
 OPTION(rgw_put_obj_max_window_size, OPT_INT, 64 * 1024 * 1024)
 OPTION(rgw_max_put_size, OPT_U64, 5ULL*1024*1024*1024)
+OPTION(rgw_max_put_param_size, OPT_U64, 1 * 1024 * 1024) // max input size for PUT requests accepting json/xml params
 
 /**
  * override max bucket index shards in zone configuration (if not zero)

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index a6eff0e710df00838410c4ea0eb9484184d83618..c2e4dff8522aa22c3fb6c7149128809ffe354727 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1065,7 +1065,7 @@ public:
 
 class RGWPutACLs : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   ACLOwner owner;
 
@@ -1108,7 +1108,7 @@ public:
 
 class RGWPutLC : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   string cookie;
 
@@ -1524,7 +1524,7 @@ public:
 class RGWDeleteMultiObj : public RGWOp {
 protected:
   int max_to_delete;
-  size_t len;
+  int len;
   char *data;
   rgw_bucket bucket;
   bool quiet;

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index c3c44fa4d04bf854238c9b43174b319067541c16..1330911d426e98e044bfee02221ed3ec21ae2f69 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1253,55 +1253,19 @@ int RGWPostObj_ObjStore::verify_params()
 
 int RGWPutACLs_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
 int RGWPutLC_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
-static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read)
+static int read_all_chunked_input(req_state *s, char **pdata, int *plen, const uint64_t max_read)
 {
 #define READ_CHUNK 4096
 #define MAX_READ_CHUNK (128 * 1024)
@@ -1325,7 +1289,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
       if (need_to_read < MAX_READ_CHUNK)
        need_to_read *= 2;
 
-      if (total > max_read) {
+      if ((unsigned)total > max_read) {
        free(data);
        return -ERANGE;
       }
@@ -1351,7 +1315,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
 }
 
 int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
-                           int max_len)
+                           const uint64_t max_len, const bool allow_chunked)
 {
   size_t cl = 0;
   int len = 0;
@@ -1359,6 +1323,9 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
 
   if (s->length)
     cl = atoll(s->length);
+  else if (!allow_chunked)
+    return -ERR_LENGTH_REQUIRED;
+
   if (cl) {
     if (cl > (size_t)max_len) {
       return -ERANGE;
@@ -1373,7 +1340,7 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
       return len;
     }
     data[len] = '\0';
-  } else if (!s->length) {
+  } else if (allow_chunked && !s->length) {
     const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
     if (!encoding || strcmp(encoding, "chunked") != 0)
       return -ERR_LENGTH_REQUIRED;
@@ -1461,28 +1428,8 @@ int RGWDeleteMultiObj_ObjStore::get_params()
   // everything is probably fine, set the bucket
   bucket = s->bucket;
 
-  size_t cl = 0;
-
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-      op_ret = -ENOMEM;
-      return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      op_ret = read_len;
-      return op_ret;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    return -EINVAL;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 6e0a87ea4cbf0c673450289a83e190d67c53bada..765c0b93addb0740b45a9855cf83ab735015bb16 100644 (file)
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -29,11 +29,11 @@ extern void rgw_flush_formatter(struct req_state *s,
                                ceph::Formatter *formatter);
 
 extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen,
-                                  int max_len);
+                                  uint64_t max_len, bool allow_chunked=true);
 
 template <class T>
 int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
-                           int max_len, bool *empty)
+                           uint64_t max_len, bool *empty)
 {
   int rv, data_len;
   char *data;
@@ -72,7 +72,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
 }
 
 template <class T>
-int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len)
+int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, uint64_t max_len, char **pdata, int *len)
 {
   int rv, data_len;
   char *data;

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 277ff62c25660b48c30fb11553581fd8c4d268c3..1b3ff56da4c6a5e2fe157c3e6b0d5e3b7718e8a8 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -733,12 +733,10 @@ public:
 
 int RGWSetBucketVersioning_ObjStore_S3::get_params()
 {
-#define GET_BUCKET_VERSIONING_BUF_MAX (128 * 1024)
-
   char *data = nullptr;
   int len = 0;
   int r =
-    rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_VERSIONING_BUF_MAX);
+    rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
   if (r < 0) {
     return r;
   }
@@ -786,11 +784,11 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response()
 
 int RGWSetBucketWebsite_ObjStore_S3::get_params()
 {
-  static constexpr uint32_t GET_BUCKET_WEBSITE_BUF_MAX = (128 * 1024);
-
   char *data = nullptr;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }
@@ -959,8 +957,10 @@ int RGWCreateBucket_ObjStore_S3::get_params()
 
   int len = 0;
   char *data = nullptr;
-#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */
-  op_ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN);
+
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
     return op_ret;
 
@@ -2425,53 +2425,38 @@ void RGWGetCORS_ObjStore_S3::send_response()
 int RGWPutCORS_ObjStore_S3::get_params()
 {
   int r;
-  char *data = NULL;
+  char *data = nullptr;
   int len = 0;
-  size_t cl = 0;
   RGWCORSXMLParser_S3 parser(s->cct);
   RGWCORSConfiguration_S3 *cors_config;
 
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       r = -ENOMEM;
-       goto done_err;
-    }
-    len = recv_body(s, data, cl);
-    if (len < 0) {
-      r = len;
-      goto done_err;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+  if (r < 0) {
+    return r;
   }
 
+  auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
+
   if (s->aws4_auth_needs_complete) {
-    int ret_auth = do_aws4_auth_completion();
-    if (ret_auth < 0) {
-      r = ret_auth;
-      goto done_err;
+    r = do_aws4_auth_completion();
+    if (r < 0) {
+      return r;
     }
   }
 
   if (!parser.init()) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (!data || !parser.parse(data, len, 1)) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
   cors_config =
     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
                                             "CORSConfiguration"));
   if (!cors_config) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
@@ -2482,11 +2467,7 @@ int RGWPutCORS_ObjStore_S3::get_params()
 
   cors_config->encode(cors_bl);
 
-  free(data);
   return 0;
-done_err:
-  free(data);
-  return r;
 }
 
 void RGWPutCORS_ObjStore_S3::send_response()
@@ -2580,10 +2561,11 @@ public:
 
 int RGWSetRequestPayment_ObjStore_S3::get_params()
 {
-#define GET_REQUEST_PAYMENT_BUF_MAX (128 * 1024)
   char *data;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_REQUEST_PAYMENT_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }