rgw/rest: round-robin resolved endpoint IPs into curl CONNECT_TO mapping

Add logic to select an IP for a given endpoint URL (RR over resolved addresses)
and build a host:port:ip:port mapping.

Store the mapping in the RGWEndpoint so the HTTP layer can apply it per request.

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>

diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 95853bacc28dc92025aa1fba988557de50de3604..71996fcb401e1a26e1783d0996f3b6fcb0f697f6 100644 (file)
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -63,19 +63,23 @@ void RGWRESTConn::resolve_endpoints() {
     boost::system::error_code ec;
     auto results = resolver.resolve(res_ep.host, "", ec);
     if (!ec && !results.empty()) {
+      std::string port_str = std::to_string(res_ep.port);
+      std::string host_port_prefix = res_ep.host + ":" + port_str + ":";
+
       for (const auto& entry : results) {
         auto ip_str = entry.endpoint().address().to_string();
         res_ep.ips.push_back(ip_str);
-        ldout(cct, 1) << "endpoint_url=" << ep_url << " resolved to ip=" << ip_str << dendl;
+        res_ep.connect_to_strings.emplace_back(host_port_prefix + ip_str + ":" + port_str);
+        ldout(cct, 2) << "endpoint_url=" << ep_url << " resolved to ip=" << ip_str << dendl;
       }
-      ldout(cct, 1) << "endpoint=" << ep_url << " resolved to "
+      ldout(cct, 2) << "endpoint=" << ep_url << " resolved to "
                 << res_ep.ips.size() << " IP addresses" << dendl;
     } else {
       ldout(cct, 0) << "WARNING: RGWRESTConn no IP addresses found for endpoint=" << ep_url
                     << (ec ? " err=" + ec.message() : "") << dendl;
     }
 
-    resolved_endpoints.push_back(std::move(res_ep));
+    resolved_endpoints.emplace(ep_url, std::move(res_ep));
   }
 }
 
@@ -155,6 +159,24 @@ RGWRESTConn& RGWRESTConn::operator=(RGWRESTConn&& other)
   return *this;
 }
 
+void RGWRESTConn::get_connect_to_mapping_for_url(RGWEndpoint& endpoint)
+{
+  if (!cct->_conf->rgw_rest_conn_connect_to_resolved_ips) {
+    return;
+  }
+
+  std::string connect_to;
+
+  auto it = resolved_endpoints.find(endpoint.get_url());
+  if (it != resolved_endpoints.end() && !it->second.connect_to_strings.empty()) {
+    auto& res_ep = it->second;
+    size_t idx = res_ep.rr_index++;
+    connect_to = res_ep.connect_to_strings[idx % res_ep.connect_to_strings.size()];
+  }
+
+  endpoint.set_connect_to(connect_to);
+}
+
 int RGWRESTConn::get_endpoint(RGWEndpoint& endpoint)
 {
   if (endpoints.empty()) {
@@ -201,7 +223,9 @@ int RGWRESTConn::get_endpoint(RGWEndpoint& endpoint)
     ldout(cct, 5) << "ERROR: no valid endpoint" << dendl;
     return -EINVAL;
   }
-  ldout(cct, 20) << "get_endpoint picked url=" << endpoint.get_url()
+
+  get_connect_to_mapping_for_url(endpoint);
+  ldout(cct, 20) << "get_endpoint picked endpoint url=" << endpoint.get_url()
     << " connect_to=" << endpoint.get_connect_to() << dendl;
 
   return 0;

diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 0dc0914a3be7bb76af03b899d2210433ee7bcfd5..f29151c97c726416e2a0634c2265258ab57ae9c2 100644 (file)
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -71,6 +71,7 @@ struct ResolvedEndpoint {
   std::string host;               // e.g., "s3.abc.com"
   int port = -1;                  // e.g., 443
   std::vector<std::string> ips; // the IPs the endpoint resolves to
+  std::vector<std::string> connect_to_strings;  // Pre-computed full connect_to strings for each IP
   size_t rr_index = 0;            // round-robin index for IPs
 };
 
@@ -81,7 +82,7 @@ class RGWRESTConn
 
   CephContext *cct;
   std::vector<std::string> endpoints;
-  std::vector<ResolvedEndpoint> resolved_endpoints;
+  std::unordered_map<std::string, ResolvedEndpoint> resolved_endpoints;
   endpoint_status_map endpoints_status;
   RGWAccessKey key;
   std::string self_zone_group;
@@ -115,7 +116,7 @@ public:
 
   int get_endpoint(RGWEndpoint& endpoint);
   RGWEndpoint get_endpoint();
-  const std::vector<ResolvedEndpoint>& get_resolved_endpoints() const { return resolved_endpoints; }
+  const std::unordered_map<std::string, ResolvedEndpoint>& get_resolved_endpoints() const { return resolved_endpoints; }
   void set_endpoint_unconnectable(const RGWEndpoint& endpoint);
   const std::string& get_self_zonegroup() {
     return self_zone_group;
@@ -162,6 +163,9 @@ public:
                        RGWRESTStreamS3PutObj *req, std::string& etag,
                        ceph::real_time *mtime, optional_yield y);
 
+  /* pick an IP to 'connect-to' given the endpoint url */
+  void get_connect_to_mapping_for_url(RGWEndpoint& endpoint);
+
   struct get_obj_params {
     const rgw_owner *uid{nullptr};
     const rgw_user *perm_check_uid{nullptr};