rgw/rest: resolve multisite endpoints to all A/AAAA records (optional)

When rgw_resolve_endpoints_into_all_addresses=true, parse each configured
endpoint URL, extract host/port, and resolve the host into all IP addresses.

Store resolution results (including a round-robin index) alongside the original
endpoint URL for later selection.

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>

diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 0f32cc201f161be91fbd6c95d32e42a661c1b984..95853bacc28dc92025aa1fba988557de50de3604 100644 (file)
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -5,11 +5,80 @@
 #include "rgw_rest_conn.h"
 #include "rgw_http_errors.h"
 #include "rgw_sal.h"
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
+
+#include <boost/url.hpp>
 
 #define dout_subsys ceph_subsys_rgw
 
 using namespace std;
 
+void RGWRESTConn::resolve_endpoints() {
+  resolved_endpoints.reserve(endpoints.size());
+
+  for (const auto& ep_url : endpoints) {
+    ResolvedEndpoint res_ep;
+    res_ep.url = ep_url;
+
+    // parse URL
+    boost::system::result<boost::urls::url_view> r = boost::urls::parse_uri(ep_url);
+    if (r.has_error()) {
+      ldout(cct, 0) << "RGWRESTConn: invalid endpoint url=" << ep_url
+                    << " err=" << r.error().message() << dendl;
+      continue;
+    }
+    boost::urls::url_view u = r.value();
+
+    // scheme
+    std::string scheme = std::string(u.scheme());
+    if (scheme.empty()) {
+      scheme = "http";
+    }
+    res_ep.scheme = scheme;
+
+    // host
+    res_ep.host = std::string(u.host());
+    if (res_ep.host.empty()) {
+      ldout(cct, 0) << "RGWRESTConn: endpoint url=" << ep_url
+                    << " has empty host" << dendl;
+      continue;
+    }
+
+    // port
+    if (u.has_port()) {
+      try {
+        res_ep.port = static_cast<uint16_t>(std::stoi(std::string(u.port())));
+      } catch (...) {
+        ldout(cct, 0) << "RGWRESTConn: invalid port in endpoint url=" << ep_url<< dendl;
+        continue;
+      }
+    } else {
+      res_ep.port = (scheme == "https" ? 443 : 80);
+    }
+
+    // resolve all IP addresses for the host
+    boost::asio::io_context io_ctx;
+    boost::asio::ip::tcp::resolver resolver(io_ctx);
+    boost::system::error_code ec;
+    auto results = resolver.resolve(res_ep.host, "", ec);
+    if (!ec && !results.empty()) {
+      for (const auto& entry : results) {
+        auto ip_str = entry.endpoint().address().to_string();
+        res_ep.ips.push_back(ip_str);
+        ldout(cct, 1) << "endpoint_url=" << ep_url << " resolved to ip=" << ip_str << dendl;
+      }
+      ldout(cct, 1) << "endpoint=" << ep_url << " resolved to "
+                << res_ep.ips.size() << " IP addresses" << dendl;
+    } else {
+      ldout(cct, 0) << "WARNING: RGWRESTConn no IP addresses found for endpoint=" << ep_url
+                    << (ec ? " err=" + ec.message() : "") << dendl;
+    }
+
+    resolved_endpoints.push_back(std::move(res_ep));
+  }
+}
+
 RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Driver* driver,
                          const string& _remote_id,
                          const list<string>& remote_endpoints,
@@ -26,6 +95,9 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Driver* driver,
                 [this](const auto& url) {
                   this->endpoints_status.emplace(url, ceph::real_clock::zero());
                 });
+  if (cct->_conf->rgw_rest_conn_connect_to_resolved_ips) {
+    resolve_endpoints();
+  }
 
   if (driver) {
     key = driver->get_zone()->get_system_key();
@@ -53,11 +125,15 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct,
                 [this](const auto& url) {
                   this->endpoints_status.emplace(url, ceph::real_clock::zero());
                 });
+  if (cct->_conf->rgw_rest_conn_connect_to_resolved_ips) {
+    resolve_endpoints();
+  }
 }
 
 RGWRESTConn::RGWRESTConn(RGWRESTConn&& other)
   : cct(other.cct),
     endpoints(std::move(other.endpoints)),
+    resolved_endpoints(std::move(other.resolved_endpoints)),
     endpoints_status(std::move(other.endpoints_status)),
     key(std::move(other.key)),
     self_zone_group(std::move(other.self_zone_group)),
@@ -70,6 +146,7 @@ RGWRESTConn& RGWRESTConn::operator=(RGWRESTConn&& other)
 {
   cct = other.cct;
   endpoints = std::move(other.endpoints);
+  resolved_endpoints = std::move(other.resolved_endpoints);
   endpoints_status = std::move(other.endpoints_status);
   key = std::move(other.key);
   self_zone_group = std::move(other.self_zone_group);

diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 5042add9d5079c0c6c6d886f6df1b7d031b96341..0dc0914a3be7bb76af03b899d2210433ee7bcfd5 100644 (file)
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -65,6 +65,15 @@ inline param_vec_t make_param_list(const std::map<std::string, std::string> *pp)
   return params;
 }
 
+struct ResolvedEndpoint {
+  std::string url;                // e.g., "https://s3.abc.com:8443"
+  std::string scheme;             // e.g., "https"
+  std::string host;               // e.g., "s3.abc.com"
+  int port = -1;                  // e.g., 443
+  std::vector<std::string> ips; // the IPs the endpoint resolves to
+  size_t rr_index = 0;            // round-robin index for IPs
+};
+
 class RGWRESTConn
 {
   /* the endpoint is not able to connect if the timestamp is not real_clock::zero */
@@ -72,6 +81,7 @@ class RGWRESTConn
 
   CephContext *cct;
   std::vector<std::string> endpoints;
+  std::vector<ResolvedEndpoint> resolved_endpoints;
   endpoint_status_map endpoints_status;
   RGWAccessKey key;
   std::string self_zone_group;
@@ -80,6 +90,8 @@ class RGWRESTConn
   HostStyle host_style;
   std::atomic<int64_t> counter = { 0 };
 
+  void resolve_endpoints(void);
+
 public:
 
   RGWRESTConn(CephContext *_cct,
@@ -103,6 +115,7 @@ public:
 
   int get_endpoint(RGWEndpoint& endpoint);
   RGWEndpoint get_endpoint();
+  const std::vector<ResolvedEndpoint>& get_resolved_endpoints() const { return resolved_endpoints; }
   void set_endpoint_unconnectable(const RGWEndpoint& endpoint);
   const std::string& get_self_zonegroup() {
     return self_zone_group;