From b2473066543c22793fb1fdc7ecc931d8b1949b6a Mon Sep 17 00:00:00 2001 From: kchheda3 Date: Wed, 26 Feb 2025 10:36:15 -0500 Subject: [PATCH] rgw/frontend: Allow multiple rgw's to run on same port on same host. The idea here is to leverage the `SO_REUSEPORT` param supported by kernel 3.9 and above, which allows the multiple services to run on the same port. Signed-off-by: kchheda3 --- PendingReleaseNotes | 2 ++ doc/radosgw/frontends.rst | 11 +++++++++++ src/rgw/rgw_asio_frontend.cc | 22 ++++++++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 404d2d7f8cdec..4e1b7f2711a99 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -21,6 +21,8 @@ * RGW: Adding missing quotes to the ETag values returned by S3 CopyPart, PostObject and CompleteMultipartUpload responses. * RGW: Added support for S3 GetObjectAttributes. +* RGW: Added BEAST frontend option 'so_reuseport' which facilitates running multiple + RGW instances on the same host by sharing a single TCP port. * RBD: All Python APIs that produce timestamps now return "aware" `datetime` objects instead of "naive" ones (i.e. those including time zone information diff --git a/doc/radosgw/frontends.rst b/doc/radosgw/frontends.rst index 5c6900f54489d..bf46e3cc15a11 100644 --- a/doc/radosgw/frontends.rst +++ b/doc/radosgw/frontends.rst @@ -135,6 +135,17 @@ Options :Default: ``16384`` :Maximum: ``65536`` +``so_reuseport`` + +:Description: If set allows multiple RGW instances on a host to listen on the same TCP port. + + ``1`` Enable running multiple RGW on same port. + + ``0`` Disallow running multiple RGW on same port. + +:Type: Integer (0 or 1) +:Default: 0 + Generic Options =============== diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc index a76aebccfd06e..e33e5b3bf789f 100644 --- a/src/rgw/rgw_asio_frontend.cc +++ b/src/rgw/rgw_asio_frontend.cc @@ -696,8 +696,12 @@ int AsioFrontend::init() l.use_nodelay = (nodelay->second == "1"); } } - + bool reuse_port = false; + auto reuse_port_it = config.find("so_reuseport"); + if (reuse_port_it != config.end()) { + reuse_port = (reuse_port_it->second == "1"); + } bool socket_bound = false; // start listeners for (auto& l : listeners) { @@ -722,7 +726,21 @@ int AsioFrontend::init() } } - l.acceptor.set_option(tcp::acceptor::reuse_address(true)); + if (reuse_port) { + // setting option |SO_REUSEPORT| allows running of multiple rgw processes on + // the same port. Can read more about the implementation here. + // https://web.git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=c617f398edd4db2b8567a28e899a88f8f574798d + int one = 1; + if (setsockopt(l.acceptor.native_handle(), SOL_SOCKET, + SO_REUSEADDR | SO_REUSEPORT, &one, sizeof(one)) == -1) { + lderr(ctx()) << "setsockopt SO_REUSEADDR | SO_REUSEPORT failed:" << + dendl; + return -1; + } + } else { + l.acceptor.set_option(tcp::acceptor::reuse_address(true)); + } + l.acceptor.bind(l.endpoint, ec); if (ec) { lderr(ctx()) << "failed to bind address " << l.endpoint -- 2.39.5