test/{librbd, rgw}: retry when bind fail with port 0

there is chance that the bind() call may fail if we have another test
happen to pick the free port picked by operating system. in this case,
we just retry up to 42 times.

in theory, this change does not fully address the racing, but it should
help to alleviate this issue.

See-also: https://tracker.ceph.com/issues/57116
Signed-off-by: Kefu Chai <tchaikov@gmail.com>

diff --git a/src/test/librbd/migration/test_mock_HttpClient.cc b/src/test/librbd/migration/test_mock_HttpClient.cc
index 7a47c4f72a1e09229874ab459183d57b4f8e95b6..630df4be6eef2b7f81ad9896502e50681e32c711 100644 (file)
--- a/src/test/librbd/migration/test_mock_HttpClient.cc
+++ b/src/test/librbd/migration/test_mock_HttpClient.cc
@@ -8,6 +8,7 @@
 #include "librbd/migration/HttpClient.h"
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
+#include <unistd.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/http.hpp>
@@ -83,10 +84,29 @@ public:
     TestMockFixture::TearDown();
   }
 
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
   void create_acceptor(bool reuse) {
-    m_acceptor.emplace(*m_image_ctx->asio_engine,
+    for (int retries = 0;; retries++) {
+      try {
+       m_acceptor.emplace(*m_image_ctx->asio_engine,
                        boost::asio::ip::tcp::endpoint(
                          boost::asio::ip::tcp::v4(), m_server_port), reuse);
+       // yay!
+       break;
+      } catch (const boost::system::system_error& e) {
+       if (retries == MAX_BIND_RETRIES) {
+         throw;
+       }
+       if (e.code() != boost::system::errc::address_in_use) {
+         throw;
+       }
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
     m_server_port = m_acceptor->local_endpoint().port();
   }
 

diff --git a/src/test/rgw/test_http_manager.cc b/src/test/rgw/test_http_manager.cc
index 418c6da022c74ba0bbc72f2dd51527f4dc70c0b5..7d4fce449e28cb23bd2e0125d61f034dc1187730 100644 (file)
--- a/src/test/rgw/test_http_manager.cc
+++ b/src/test/rgw/test_http_manager.cc
@@ -15,6 +15,7 @@
 #include "rgw/rgw_http_client.h"
 #include "global/global_init.h"
 #include "common/ceph_argparse.h"
+#include <unistd.h>
 #include <curl/curl.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/write.hpp>
@@ -23,14 +24,43 @@
 
 using namespace std;
 
+namespace {
+  using tcp = boost::asio::ip::tcp;
+
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
+  tcp::acceptor try_bind(boost::asio::io_context& ioctx) {
+    using tcp = boost::asio::ip::tcp;
+    tcp::endpoint endpoint(tcp::v4(), 0);
+    tcp::acceptor acceptor(ioctx);
+    acceptor.open(endpoint.protocol());
+    for (int retries = 0;; retries++) {
+      try {
+       acceptor.bind(endpoint);
+       // yay!
+       break;
+      } catch (const boost::system::system_error& e) {
+       if (retries == MAX_BIND_RETRIES) {
+         throw;
+       }
+       if (e.code() != boost::system::errc::address_in_use) {
+         throw;
+       }
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
+    return acceptor;
+  }
+}
+
 TEST(HTTPManager, ReadTruncated)
 {
   using tcp = boost::asio::ip::tcp;
-  tcp::endpoint endpoint(tcp::v4(), 0);
   boost::asio::io_context ioctx;
-  tcp::acceptor acceptor(ioctx);
-  acceptor.open(endpoint.protocol());
-  acceptor.bind(endpoint);
+  auto acceptor = try_bind(ioctx);
   acceptor.listen();
 
   std::thread server{[&] {
@@ -54,11 +84,8 @@ TEST(HTTPManager, ReadTruncated)
 TEST(HTTPManager, Head)
 {
   using tcp = boost::asio::ip::tcp;
-  tcp::endpoint endpoint(tcp::v4(), 0);
   boost::asio::io_context ioctx;
-  tcp::acceptor acceptor(ioctx);
-  acceptor.open(endpoint.protocol());
-  acceptor.bind(endpoint);
+  auto acceptor = try_bind(ioctx);
   acceptor.listen();
 
   std::thread server{[&] {