rgw: make a configurable size for requests with xml params

We currently read all user input for a few apis accepting xml, avoid
doing this and error out early, most s3 apis limit to about 1000 xml
entries, make this configurable via a new config param,
`rgw_max_put_param_size` defaulting to 1MB. Also modified
`rgw_rest_read_all_input` with an additional param to not allow for
chunked uploads, though we error out in a better way by responding with
content length required rather than S3's 501 if chunked is set.

This also adds the same behavior in RGWPutCORS reusing
`rgw_rest_read_all_input`, and using a data_deleter to manage the char*

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
rgw: rgw_rest_s3: make PUT CORS accept only a configured max input

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
(cherry picked from commit 35375b75160dc93beeb04c7fe010dc0d21952534)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 95b8e98565100f53c024d49e577b79a5755ee0d7..d4e5c305c6c01777b1b2a080932f074b72eab3c7 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1359,6 +1359,7 @@ OPTION(nss_db_path, OPT_STR, "") // path to nss db
 
 OPTION(rgw_max_chunk_size, OPT_INT, 512 * 1024)
 OPTION(rgw_max_put_size, OPT_U64, 5ULL*1024*1024*1024)
+OPTION(rgw_max_put_param_size, OPT_U64, 1 * 1024 * 1024) // max input size for PUT requests accepting json/xml params
 
 /**
  * override max bucket index shards in zone configuration (if not zero)

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 62f4e3b838ab1e560c9628865b8ee91f0a22b75b..bd4eac4598f0823fd28c7ccd526f1ef94fe451d2 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1065,7 +1065,7 @@ public:
 
 class RGWPutACLs : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   ACLOwner owner;
 
@@ -1108,7 +1108,7 @@ public:
 
 class RGWPutLC : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   string cookie;
 
@@ -1524,7 +1524,7 @@ public:
 class RGWDeleteMultiObj : public RGWOp {
 protected:
   int max_to_delete;
-  size_t len;
+  int len;
   char *data;
   rgw_bucket bucket;
   bool quiet;

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 2ca556e20b8df4d6d753a8ed1a2bba5b43024695..c3e119c874586314d078d64770763912939a0921 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1253,55 +1253,19 @@ int RGWPostObj_ObjStore::verify_params()
 
 int RGWPutACLs_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
 int RGWPutLC_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
-static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read)
+static int read_all_chunked_input(req_state *s, char **pdata, int *plen, const uint64_t max_read)
 {
 #define READ_CHUNK 4096
 #define MAX_READ_CHUNK (128 * 1024)
@@ -1325,7 +1289,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
       if (need_to_read < MAX_READ_CHUNK)
        need_to_read *= 2;
 
-      if (total > max_read) {
+      if ((unsigned)total > max_read) {
        free(data);
        return -ERANGE;
       }
@@ -1351,7 +1315,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
 }
 
 int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
-                           int max_len)
+                           const uint64_t max_len, const bool allow_chunked)
 {
   size_t cl = 0;
   int len = 0;
@@ -1359,6 +1323,9 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
 
   if (s->length)
     cl = atoll(s->length);
+  else if (!allow_chunked)
+    return -ERR_LENGTH_REQUIRED;
+
   if (cl) {
     if (cl > (size_t)max_len) {
       return -ERANGE;
@@ -1373,7 +1340,7 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
       return len;
     }
     data[len] = '\0';
-  } else if (!s->length) {
+  } else if (allow_chunked && !s->length) {
     const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
     if (!encoding || strcmp(encoding, "chunked") != 0)
       return -ERR_LENGTH_REQUIRED;
@@ -1461,28 +1428,8 @@ int RGWDeleteMultiObj_ObjStore::get_params()
   // everything is probably fine, set the bucket
   bucket = s->bucket;
 
-  size_t cl = 0;
-
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-      op_ret = -ENOMEM;
-      return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      op_ret = read_len;
-      return op_ret;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    return -EINVAL;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 293b5268de2e6b9f810e11e3b32069533af25bfb..6657d861fb71ba3249d325d7630c490a290e3f52 100644 (file)
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -29,11 +29,11 @@ extern void rgw_flush_formatter(struct req_state *s,
                                ceph::Formatter *formatter);
 
 extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen,
-                                  int max_len);
+                                  uint64_t max_len, bool allow_chunked=true);
 
 template <class T>
 int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
-                           int max_len, bool *empty)
+                           uint64_t max_len, bool *empty)
 {
   int rv, data_len;
   char *data;
@@ -72,7 +72,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
 }
 
 template <class T>
-int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len)
+int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, uint64_t max_len, char **pdata, int *len)
 {
   int rv, data_len;
   char *data;

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index a745b95f5a2ef5a6f0a23afdebb756c33151d673..ea323a14cdc902a3e12218b8ba2593fe9edf538a 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -729,12 +729,10 @@ public:
 
 int RGWSetBucketVersioning_ObjStore_S3::get_params()
 {
-#define GET_BUCKET_VERSIONING_BUF_MAX (128 * 1024)
-
   char *data = nullptr;
   int len = 0;
   int r =
-    rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_VERSIONING_BUF_MAX);
+    rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
   if (r < 0) {
     return r;
   }
@@ -782,11 +780,11 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response()
 
 int RGWSetBucketWebsite_ObjStore_S3::get_params()
 {
-  static constexpr uint32_t GET_BUCKET_WEBSITE_BUF_MAX = (128 * 1024);
-
   char *data = nullptr;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }
@@ -955,8 +953,10 @@ int RGWCreateBucket_ObjStore_S3::get_params()
 
   int len = 0;
   char *data = nullptr;
-#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */
-  op_ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN);
+
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
     return op_ret;
 
@@ -2421,53 +2421,38 @@ void RGWGetCORS_ObjStore_S3::send_response()
 int RGWPutCORS_ObjStore_S3::get_params()
 {
   int r;
-  char *data = NULL;
+  char *data = nullptr;
   int len = 0;
-  size_t cl = 0;
   RGWCORSXMLParser_S3 parser(s->cct);
   RGWCORSConfiguration_S3 *cors_config;
 
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       r = -ENOMEM;
-       goto done_err;
-    }
-    len = recv_body(s, data, cl);
-    if (len < 0) {
-      r = len;
-      goto done_err;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+  if (r < 0) {
+    return r;
   }
 
+  auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
+
   if (s->aws4_auth_needs_complete) {
-    int ret_auth = do_aws4_auth_completion();
-    if (ret_auth < 0) {
-      r = ret_auth;
-      goto done_err;
+    r = do_aws4_auth_completion();
+    if (r < 0) {
+      return r;
     }
   }
 
   if (!parser.init()) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (!data || !parser.parse(data, len, 1)) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
   cors_config =
     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
                                             "CORSConfiguration"));
   if (!cors_config) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
@@ -2478,11 +2463,7 @@ int RGWPutCORS_ObjStore_S3::get_params()
 
   cors_config->encode(cors_bl);
 
-  free(data);
   return 0;
-done_err:
-  free(data);
-  return r;
 }
 
 void RGWPutCORS_ObjStore_S3::send_response()
@@ -2576,10 +2557,11 @@ public:
 
 int RGWSetRequestPayment_ObjStore_S3::get_params()
 {
-#define GET_REQUEST_PAYMENT_BUF_MAX (128 * 1024)
   char *data;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_REQUEST_PAYMENT_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }