rgw: reuse CURL* for keystone.

When using keystone with https (the recommended setting),
it is inefficient to start up a new SSL connection for
each and every operation.  Keeping a CURL* structure around
should reduce the cost of doing this.  This logic tries
to do so, but it also tries to free them fairly aggressively
(5-10 seconds).  This should still greatly reduce load on
keystone at peak times while not tying up excess resources.

Fixes: https://tracker.ceph.com/issues/23162
Signed-off-by: Marcus Watts <mwatts@redhat.com>
(cherry picked from commit 0c5cee1977eac232045571f4850e64be93271aec)

 Conflicts:
  src/rgw/rgw_common.h
  trivial conflict as lowercase_dash_http_attr isnt a part of L

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 0beac055e366e57b15a505dad051357176dd62f0..dc39f10fbb3b353182fb581ddc070a09e6b3f381 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1909,4 +1909,7 @@ extern string  calc_hash_sha256_close_stream(SHA256 **hash);
 
 extern int rgw_parse_op_type_list(const string& str, uint32_t *perm);
 
+void rgw_setup_saved_curl_handles();
+void rgw_release_all_curl_handles();
+
 #endif

diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index 6edb5dca6f763c661517a84062ef68bcae552c30..770580ef56a33a75913a050c6914d5cf8556d6a9 100644 (file)
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -75,6 +75,135 @@ struct rgw_http_req_data : public RefCountedObject {
   }
 };
 
+struct RGWCurlHandle {
+  int uses;
+  mono_time lastuse;
+  CURL* h;
+
+  RGWCurlHandle(CURL* h) : uses(0), h(h) {};
+  CURL* operator*() {
+    return this->h;
+  }
+};
+
+#define MAXIDLE 5
+class RGWCurlHandles : public Thread {
+public:
+  Mutex cleaner_lock;
+  std::vector<RGWCurlHandle*>saved_curl;
+  int cleaner_shutdown;
+  Cond cleaner_cond;
+
+  RGWCurlHandles() :
+    cleaner_lock{"RGWCurlHandles::cleaner_lock"},
+    cleaner_shutdown{0} {
+  }
+
+  RGWCurlHandle* get_curl_handle();
+  void release_curl_handle_now(RGWCurlHandle* curl);
+  void release_curl_handle(RGWCurlHandle* curl);
+  void flush_curl_handles();
+  void* entry();
+  void stop();
+};
+
+RGWCurlHandle* RGWCurlHandles::get_curl_handle() {
+  RGWCurlHandle* curl = 0;
+  CURL* h;
+  {
+    Mutex::Locker lock(cleaner_lock);
+    if (!saved_curl.empty()) {
+      curl = *saved_curl.begin();
+      saved_curl.erase(saved_curl.begin());
+    }
+  }
+  if (curl) {
+  } else if ((h = curl_easy_init())) {
+    curl = new RGWCurlHandle{h};
+  } else {
+    // curl = 0;
+  }
+  return curl;
+}
+
+void RGWCurlHandles::release_curl_handle_now(RGWCurlHandle* curl)
+{
+  curl_easy_cleanup(**curl);
+  delete curl;
+}
+
+void RGWCurlHandles::release_curl_handle(RGWCurlHandle* curl)
+{
+  if (cleaner_shutdown) {
+    release_curl_handle_now(curl);
+  } else {
+    curl_easy_reset(**curl);
+    Mutex::Locker lock(cleaner_lock);
+    curl->lastuse = mono_clock::now();
+    saved_curl.insert(saved_curl.begin(), 1, curl);
+  }
+}
+
+void* RGWCurlHandles::entry()
+{
+  RGWCurlHandle* curl;
+  Mutex::Locker lock(cleaner_lock);
+
+  for (;;) {
+    if (cleaner_shutdown) {
+      if (saved_curl.empty())
+        break;
+    } else {
+      utime_t release = ceph_clock_now(g_ceph_context) + utime_t(MAXIDLE,0);
+      cleaner_cond.WaitUntil(cleaner_lock, release);
+    }
+    mono_time now = mono_clock::now();
+    while (!saved_curl.empty()) {
+      auto cend = saved_curl.end();
+      --cend;
+      curl = *cend;
+      if (!cleaner_shutdown && now - curl->lastuse < std::chrono::seconds(MAXIDLE))
+        break;
+      saved_curl.erase(cend);
+      release_curl_handle_now(curl);
+    }
+  }
+  return nullptr;
+}
+
+void RGWCurlHandles::stop()
+{
+  Mutex::Locker lock(cleaner_lock);
+  cleaner_shutdown = 1;
+  cleaner_cond.Signal();
+}
+
+void RGWCurlHandles::flush_curl_handles()
+{
+  stop();
+  join();
+  if (!saved_curl.empty()) {
+    dout(0) << "ERROR: " << __func__ << " failed final cleanup" << dendl;
+  }
+  saved_curl.shrink_to_fit();
+}
+
+static RGWCurlHandles *handles;
+// XXX make this part of the token cache?  (but that's swift-only;
+//     and this especially needs to integrates with s3...)
+
+void rgw_setup_saved_curl_handles()
+{
+  handles = new RGWCurlHandles();
+  handles->create("rgw_curl");
+}
+
+void rgw_release_all_curl_handles()
+{
+  handles->flush_curl_handles();
+  delete handles;
+}
+
 /*
  * the simple set of callbacks will be called on RGWHTTPClient::process()
  */
@@ -226,7 +355,8 @@ int RGWHTTPClient::process(const char *method, const char *url)
   last_method = (method ? method : "");
   last_url = (url ? url : "");
 
-  curl_handle = curl_easy_init();
+  auto ca = handles->get_curl_handle();
+  curl_handle = **ca;
 
   dout(20) << "sending request to " << url << dendl;
 
@@ -264,7 +394,7 @@ int RGWHTTPClient::process(const char *method, const char *url)
     ret = -EINVAL;
   }
   curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_status);
-  curl_easy_cleanup(curl_handle);
+  handles->release_curl_handle(ca);
   curl_slist_free_all(h);
 
   return ret;

diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index fd821bfcf81c53d68d599af8f39e59f98690cefb..d96d97b0ca87d30fd8f495f2a12ad0bfc1b45af6 100644 (file)
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -295,6 +295,7 @@ int main(int argc, const char **argv)
   rgw_init_resolver();
   
   curl_global_init(CURL_GLOBAL_ALL);
+  rgw_setup_saved_curl_handles();
   
   FCGX_Init();
 
@@ -525,6 +526,7 @@ int main(int argc, const char **argv)
 
   rgw_tools_cleanup();
   rgw_shutdown_resolver();
+  rgw_release_all_curl_handles();
   curl_global_cleanup();
 
   rgw_perf_stop(g_ceph_context);