test/{librbd, rgw}: retry when bind fail with port 0

there is chance that the bind() call may fail if we have another test
happen to pick the free port picked by operating system. in this case,
we just retry up to 42 times.

in theory, this change does not fully address the racing, but it should
help to alleviate this issue.

See-also: https://tracker.ceph.com/issues/57116
Signed-off-by: Kefu Chai <tchaikov@gmail.com>
(cherry picked from commit aa7885f7cc41390fcc8eeb82bc7142c3ff6a53f9)

Conflicts:
	src/test/rgw/test_http_manager.cc [ commit f5019d2a8388 ("rgw:
	  Set CURLOPT_NOBODY for HEAD request") not in pacific ]

diff --git a/src/test/librbd/migration/test_mock_HttpClient.cc b/src/test/librbd/migration/test_mock_HttpClient.cc
index 7a47c4f72a1e09229874ab459183d57b4f8e95b6..630df4be6eef2b7f81ad9896502e50681e32c711 100644 (file)
--- a/src/test/librbd/migration/test_mock_HttpClient.cc
+++ b/src/test/librbd/migration/test_mock_HttpClient.cc
@@ -8,6 +8,7 @@
 #include "librbd/migration/HttpClient.h"
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
+#include <unistd.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/http.hpp>
@@ -83,10 +84,29 @@ public:
     TestMockFixture::TearDown();
   }
 
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
   void create_acceptor(bool reuse) {
-    m_acceptor.emplace(*m_image_ctx->asio_engine,
+    for (int retries = 0;; retries++) {
+      try {
+       m_acceptor.emplace(*m_image_ctx->asio_engine,
                        boost::asio::ip::tcp::endpoint(
                          boost::asio::ip::tcp::v4(), m_server_port), reuse);
+       // yay!
+       break;
+      } catch (const boost::system::system_error& e) {
+       if (retries == MAX_BIND_RETRIES) {
+         throw;
+       }
+       if (e.code() != boost::system::errc::address_in_use) {
+         throw;
+       }
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
     m_server_port = m_acceptor->local_endpoint().port();
   }
 

diff --git a/src/test/rgw/test_http_manager.cc b/src/test/rgw/test_http_manager.cc
index e27e1895a45c354158dc0cc194a83493dc145423..6aff557bd02e358411489f06b8837e1b14566455 100644 (file)
--- a/src/test/rgw/test_http_manager.cc
+++ b/src/test/rgw/test_http_manager.cc
@@ -15,20 +15,50 @@
 #include "rgw/rgw_http_client.h"
 #include "global/global_init.h"
 #include "common/ceph_argparse.h"
+#include <unistd.h>
 #include <curl/curl.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/write.hpp>
 #include <thread>
 #include <gtest/gtest.h>
 
+namespace {
+  using tcp = boost::asio::ip::tcp;
+
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
+  tcp::acceptor try_bind(boost::asio::io_context& ioctx) {
+    using tcp = boost::asio::ip::tcp;
+    tcp::endpoint endpoint(tcp::v4(), 0);
+    tcp::acceptor acceptor(ioctx);
+    acceptor.open(endpoint.protocol());
+    for (int retries = 0;; retries++) {
+      try {
+       acceptor.bind(endpoint);
+       // yay!
+       break;
+      } catch (const boost::system::system_error& e) {
+       if (retries == MAX_BIND_RETRIES) {
+         throw;
+       }
+       if (e.code() != boost::system::errc::address_in_use) {
+         throw;
+       }
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
+    return acceptor;
+  }
+}
+
 TEST(HTTPManager, ReadTruncated)
 {
   using tcp = boost::asio::ip::tcp;
-  tcp::endpoint endpoint(tcp::v4(), 0);
   boost::asio::io_context ioctx;
-  tcp::acceptor acceptor(ioctx);
-  acceptor.open(endpoint.protocol());
-  acceptor.bind(endpoint);
+  auto acceptor = try_bind(ioctx);
   acceptor.listen();
 
   std::thread server{[&] {