From: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
Date: Fri, 8 Dec 2017 01:05:30 +0000 (+0800)
Subject: rgw: add curl_low_speed_limit and curl_low_speed_time config to avoid
X-Git-Tag: v13.2.2~93^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8f646c568849584f1b728a9ced06754385c6ea0e;p=ceph.git

rgw: add curl_low_speed_limit and curl_low_speed_time config to avoid
the thread hangs in data sync.

Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>

rgw: mitigate NIC problems like X710 transmit queue hang

Continuation of #19282
- change the default speed limit to 1KBps
- fix options description spacing

Signed-off-by: Mark Kogan <mkogan@redhat.com>
(cherry picked from commit f4171225fa479b19c3ec5ff51943c84fd53f85d1)
---

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index ee642b1ca9eb..1028d8c9c8da 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1421,6 +1421,8 @@ OPTION(rgw_md_log_max_shards, OPT_INT) // max shards for metadata log
 OPTION(rgw_num_zone_opstate_shards, OPT_INT) // max shards for keeping inter-region copy progress info
 OPTION(rgw_opstate_ratelimit_sec, OPT_INT) // min time between opstate updates on a single upload (0 for disabling ratelimit)
 OPTION(rgw_curl_wait_timeout_ms, OPT_INT) // timeout for certain curl calls
+OPTION(rgw_curl_low_speed_limit, OPT_INT) // low speed limit for certain curl calls
+OPTION(rgw_curl_low_speed_time, OPT_INT) // low speed time for certain curl calls
 OPTION(rgw_copy_obj_progress, OPT_BOOL) // should dump progress during long copy operations?
 OPTION(rgw_copy_obj_progress_every_bytes, OPT_INT) // min bytes between copy progress output
 OPTION(rgw_obj_tombstone_cache_size, OPT_INT) // how many objects in tombstone cache, which is used in multi-zone sync to keep
diff --git a/src/common/options.cc b/src/common/options.cc
index 35fade3fd636..2bcd77431263 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5724,6 +5724,20 @@ std::vector<Option> get_rgw_options() {
     .set_default(1000)
     .set_description(""),
 
+    Option("rgw_curl_low_speed_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(1024)
+    .set_long_description(
+        "It contains the average transfer speed in bytes per second that the "
+        "transfer should be below during rgw_curl_low_speed_time seconds for libcurl "
+        "to consider it to be too slow and abort. Set it zero to disable this."),
+
+    Option("rgw_curl_low_speed_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(30)
+    .set_long_description(
+        "It contains the time in number seconds that the transfer speed should be below "
+        "the rgw_curl_low_speed_limit for the library to consider it too slow and abort. "
+        "Set it zero to disable this."),
+
     Option("rgw_copy_obj_progress", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Send progress report through copy operation")
diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index f341f87281cf..969b5cd8c725 100644
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -493,6 +493,8 @@ int RGWHTTPClient::init_request(rgw_http_req_data *_req_data, bool send_data_hin
   curl_easy_setopt(easy_handle, CURLOPT_WRITEFUNCTION, receive_http_data);
   curl_easy_setopt(easy_handle, CURLOPT_WRITEDATA, (void *)req_data);
   curl_easy_setopt(easy_handle, CURLOPT_ERRORBUFFER, (void *)req_data->error_buf);
+  curl_easy_setopt(easy_handle, CURLOPT_LOW_SPEED_TIME, cct->_conf->rgw_curl_low_speed_time);
+  curl_easy_setopt(easy_handle, CURLOPT_LOW_SPEED_LIMIT, cct->_conf->rgw_curl_low_speed_limit);
   if (h) {
     curl_easy_setopt(easy_handle, CURLOPT_HTTPHEADER, (void *)h);
   }
@@ -1149,6 +1151,9 @@ void *RGWHTTPManager::reqs_thread_entry()
         switch (result) {
           case CURLE_OK:
             break;
+          case CURLE_OPERATION_TIMEDOUT:
+            dout(0) << "WARNING: curl operation timed out, network average transfer speed less than " 
+              << cct->_conf->rgw_curl_low_speed_limit << " Bytes per second during " << cct->_conf->rgw_curl_low_speed_time << " seconds." << dendl;
           default:
             dout(20) << "ERROR: msg->data.result=" << result << " req_data->id=" << id << " http_status=" << http_status << dendl;
 	    break;