rgw/rgw_http_client.cc: Introduce a new configuration item - rgw_curl_buffersize

Config this item to a large value will aggressively enhance large object
sync performance. So raise the default value of rgw_curl_buffersize from 16KB
to 512KB, which is also the buffer size used for compression in #21479

Signed-off-by: Yang Honggang <yanghonggang@kuaishou.com>
Signed-off-by: Ruan Zitao <ruanzitao@kuaishou.com>

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index dd5b3e0af2f0aaa4671b5f114b582eac3735b4c2..0be601e8ec144ddc745ba8f7dbfcb7ac5ff9dce1 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1426,6 +1426,7 @@ OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL) // enable relaxed bucket name rule
 OPTION(rgw_defer_to_bucket_acls, OPT_STR) // if the user has bucket perms)
 OPTION(rgw_list_buckets_max_chunk, OPT_INT) // max buckets to retrieve in a single op when listing user buckets
 OPTION(rgw_md_log_max_shards, OPT_INT) // max shards for metadata log
+OPTION(rgw_curl_buffersize, OPT_INT) // set preferred receive buffer size for curl calls
 OPTION(rgw_curl_wait_timeout_ms, OPT_INT) // timeout for certain curl calls
 OPTION(rgw_curl_low_speed_limit, OPT_INT) // low speed limit for certain curl calls
 OPTION(rgw_curl_low_speed_time, OPT_INT) // low speed time for certain curl calls

diff --git a/src/common/options.cc b/src/common/options.cc
index 7de01868ccfdf6b4f1a3a94e806c73f89731e3ef..396b66df4851a360fb30a0d8ffeeb3e1c632a65d 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6495,6 +6495,14 @@ std::vector<Option> get_rgw_options() {
         "the metadata sync parallelism as a shard can only be processed by a single "
         "RGW at a time"),
 
+    Option("rgw_curl_buffersize", Option::TYPE_INT, Option::LEVEL_DEV)
+    .set_default(512_K)
+    .set_min_max(1_K, 512_K)
+    .set_long_description(
+        "Pass a long specifying your preferred size (in bytes) for the receive"
+        "buffer in libcurl. "
+        "See: https://curl.se/libcurl/c/CURLOPT_BUFFERSIZE.html"),
+
     Option("rgw_curl_wait_timeout_ms", Option::TYPE_INT, Option::LEVEL_DEV)
     .set_default(1000)
     .set_description(""),

diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index 15be26f5229beea8d7821065f62b15ffed177988..1ade7dfe8284b72c60435383f65250e1cfb42e56 100644 (file)
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -541,6 +541,7 @@ int RGWHTTPClient::init_request(rgw_http_req_data *_req_data)
   curl_easy_setopt(easy_handle, CURLOPT_LOW_SPEED_LIMIT, cct->_conf->rgw_curl_low_speed_limit);
   curl_easy_setopt(easy_handle, CURLOPT_READFUNCTION, send_http_data);
   curl_easy_setopt(easy_handle, CURLOPT_READDATA, (void *)req_data);
+  curl_easy_setopt(easy_handle, CURLOPT_BUFFERSIZE, cct->_conf->rgw_curl_buffersize);
   if (send_data_hint || is_upload_request(method)) {
     curl_easy_setopt(easy_handle, CURLOPT_UPLOAD, 1L);
   }