From 0c5cee1977eac232045571f4850e64be93271aec Mon Sep 17 00:00:00 2001 From: Marcus Watts Date: Tue, 27 Feb 2018 02:53:37 -0500 Subject: [PATCH] rgw: reuse CURL* for keystone. When using keystone with https (the recommended setting), it is inefficient to start up a new SSL connection for each and every operation. Keeping a CURL* structure around should reduce the cost of doing this. This logic tries to do so, but it also tries to free them fairly aggressively (5-10 seconds). This should still greatly reduce load on keystone at peak times while not tying up excess resources. Fixes: https://tracker.ceph.com/issues/23162 Signed-off-by: Marcus Watts --- src/rgw/rgw_common.h | 3 + src/rgw/rgw_http_client.cc | 134 ++++++++++++++++++++++++++++++++++++- src/rgw/rgw_main.cc | 2 + 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 2e11142316872..6a364502770c4 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -2376,4 +2376,7 @@ extern bool match_policy(boost::string_view pattern, boost::string_view input, extern string camelcase_dash_http_attr(const string& orig); extern string lowercase_dash_http_attr(const string& orig); +void rgw_setup_saved_curl_handles(); +void rgw_release_all_curl_handles(); + #endif diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc index f81ce94b3e039..f96b7d5c51cfc 100644 --- a/src/rgw/rgw_http_client.cc +++ b/src/rgw/rgw_http_client.cc @@ -79,6 +79,135 @@ struct rgw_http_req_data : public RefCountedObject { } }; +struct RGWCurlHandle { + int uses; + mono_time lastuse; + CURL* h; + + RGWCurlHandle(CURL* h) : uses(0), h(h) {}; + CURL* operator*() { + return this->h; + } +}; + +#define MAXIDLE 5 +class RGWCurlHandles : public Thread { +public: + Mutex cleaner_lock; + std::vectorsaved_curl; + int cleaner_shutdown; + Cond cleaner_cond; + + RGWCurlHandles() : + cleaner_lock{"RGWCurlHandles::cleaner_lock"}, + cleaner_shutdown{0} { + } + + RGWCurlHandle* get_curl_handle(); + void release_curl_handle_now(RGWCurlHandle* curl); + void release_curl_handle(RGWCurlHandle* curl); + void flush_curl_handles(); + void* entry(); + void stop(); +}; + +RGWCurlHandle* RGWCurlHandles::get_curl_handle() { + RGWCurlHandle* curl = 0; + CURL* h; + { + Mutex::Locker lock(cleaner_lock); + if (!saved_curl.empty()) { + curl = *saved_curl.begin(); + saved_curl.erase(saved_curl.begin()); + } + } + if (curl) { + } else if ((h = curl_easy_init())) { + curl = new RGWCurlHandle{h}; + } else { + // curl = 0; + } + return curl; +} + +void RGWCurlHandles::release_curl_handle_now(RGWCurlHandle* curl) +{ + curl_easy_cleanup(**curl); + delete curl; +} + +void RGWCurlHandles::release_curl_handle(RGWCurlHandle* curl) +{ + if (cleaner_shutdown) { + release_curl_handle_now(curl); + } else { + curl_easy_reset(**curl); + Mutex::Locker lock(cleaner_lock); + curl->lastuse = mono_clock::now(); + saved_curl.insert(saved_curl.begin(), 1, curl); + } +} + +void* RGWCurlHandles::entry() +{ + RGWCurlHandle* curl; + Mutex::Locker lock(cleaner_lock); + + for (;;) { + if (cleaner_shutdown) { + if (saved_curl.empty()) + break; + } else { + utime_t release = ceph_clock_now() + utime_t(MAXIDLE,0); + cleaner_cond.WaitUntil(cleaner_lock, release); + } + mono_time now = mono_clock::now(); + while (!saved_curl.empty()) { + auto cend = saved_curl.end(); + --cend; + curl = *cend; + if (!cleaner_shutdown && now - curl->lastuse < std::chrono::seconds(MAXIDLE)) + break; + saved_curl.erase(cend); + release_curl_handle_now(curl); + } + } + return nullptr; +} + +void RGWCurlHandles::stop() +{ + Mutex::Locker lock(cleaner_lock); + cleaner_shutdown = 1; + cleaner_cond.Signal(); +} + +void RGWCurlHandles::flush_curl_handles() +{ + stop(); + join(); + if (!saved_curl.empty()) { + dout(0) << "ERROR: " << __func__ << " failed final cleanup" << dendl; + } + saved_curl.shrink_to_fit(); +} + +static RGWCurlHandles *handles; +// XXX make this part of the token cache? (but that's swift-only; +// and this especially needs to integrates with s3...) + +void rgw_setup_saved_curl_handles() +{ + handles = new RGWCurlHandles(); + handles->create("rgw_curl"); +} + +void rgw_release_all_curl_handles() +{ + handles->flush_curl_handles(); + delete handles; +} + /* * the simple set of callbacks will be called on RGWHTTPClient::process() */ @@ -253,7 +382,8 @@ int RGWHTTPClient::process(const char *method, const char *url) last_method = (method ? method : ""); last_url = (url ? url : ""); - curl_handle = curl_easy_init(); + auto ca = handles->get_curl_handle(); + curl_handle = **ca; dout(20) << "sending request to " << url << dendl; @@ -291,7 +421,7 @@ int RGWHTTPClient::process(const char *method, const char *url) ret = -EINVAL; } curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_status); - curl_easy_cleanup(curl_handle); + handles->release_curl_handle(ca); curl_slist_free_all(h); return ret; diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 7456d8e7bb631..60053635fc4ae 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -296,6 +296,7 @@ int main(int argc, const char **argv) rgw_init_resolver(); curl_global_init(CURL_GLOBAL_ALL); + rgw_setup_saved_curl_handles(); #if defined(WITH_RADOSGW_FCGI_FRONTEND) FCGX_Init(); @@ -571,6 +572,7 @@ int main(int argc, const char **argv) rgw_tools_cleanup(); rgw_shutdown_resolver(); + rgw_release_all_curl_handles(); curl_global_cleanup(); rgw_perf_stop(g_ceph_context); -- 2.39.5