From aa7885f7cc41390fcc8eeb82bc7142c3ff6a53f9 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 4 Sep 2022 20:37:32 +0800
Subject: [PATCH] test/{librbd, rgw}: retry when bind fail with port 0

there is chance that the bind() call may fail if we have another test
happen to pick the free port picked by operating system. in this case,
we just retry up to 42 times.

in theory, this change does not fully address the racing, but it should
help to alleviate this issue.

See-also: https://tracker.ceph.com/issues/57116
Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 .../librbd/migration/test_mock_HttpClient.cc  | 22 +++++++++-
 src/test/rgw/test_http_manager.cc             | 43 +++++++++++++++----
 2 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/src/test/librbd/migration/test_mock_HttpClient.cc b/src/test/librbd/migration/test_mock_HttpClient.cc
index 7a47c4f72a1..630df4be6ee 100644
--- a/src/test/librbd/migration/test_mock_HttpClient.cc
+++ b/src/test/librbd/migration/test_mock_HttpClient.cc
@@ -8,6 +8,7 @@
 #include "librbd/migration/HttpClient.h"
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
+#include <unistd.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/http.hpp>
@@ -83,10 +84,29 @@ public:
     TestMockFixture::TearDown();
   }
 
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
   void create_acceptor(bool reuse) {
-    m_acceptor.emplace(*m_image_ctx->asio_engine,
+    for (int retries = 0;; retries++) {
+      try {
+	m_acceptor.emplace(*m_image_ctx->asio_engine,
                        boost::asio::ip::tcp::endpoint(
                          boost::asio::ip::tcp::v4(), m_server_port), reuse);
+	// yay!
+	break;
+      } catch (const boost::system::system_error& e) {
+	if (retries == MAX_BIND_RETRIES) {
+	  throw;
+	}
+	if (e.code() != boost::system::errc::address_in_use) {
+	  throw;
+	}
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
     m_server_port = m_acceptor->local_endpoint().port();
   }
 
diff --git a/src/test/rgw/test_http_manager.cc b/src/test/rgw/test_http_manager.cc
index 418c6da022c..7d4fce449e2 100644
--- a/src/test/rgw/test_http_manager.cc
+++ b/src/test/rgw/test_http_manager.cc
@@ -15,6 +15,7 @@
 #include "rgw/rgw_http_client.h"
 #include "global/global_init.h"
 #include "common/ceph_argparse.h"
+#include <unistd.h>
 #include <curl/curl.h>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/write.hpp>
@@ -23,14 +24,43 @@
 
 using namespace std;
 
+namespace {
+  using tcp = boost::asio::ip::tcp;
+
+  // if we have a racing where another thread manages to bind and listen the
+  // port picked by this acceptor, try again.
+  static constexpr int MAX_BIND_RETRIES = 42;
+
+  tcp::acceptor try_bind(boost::asio::io_context& ioctx) {
+    using tcp = boost::asio::ip::tcp;
+    tcp::endpoint endpoint(tcp::v4(), 0);
+    tcp::acceptor acceptor(ioctx);
+    acceptor.open(endpoint.protocol());
+    for (int retries = 0;; retries++) {
+      try {
+	acceptor.bind(endpoint);
+	// yay!
+	break;
+      } catch (const boost::system::system_error& e) {
+	if (retries == MAX_BIND_RETRIES) {
+	  throw;
+	}
+	if (e.code() != boost::system::errc::address_in_use) {
+	  throw;
+	}
+      }
+      // backoff a little bit
+      usleep(retries * 10'000);
+    }
+    return acceptor;
+  }
+}
+
 TEST(HTTPManager, ReadTruncated)
 {
   using tcp = boost::asio::ip::tcp;
-  tcp::endpoint endpoint(tcp::v4(), 0);
   boost::asio::io_context ioctx;
-  tcp::acceptor acceptor(ioctx);
-  acceptor.open(endpoint.protocol());
-  acceptor.bind(endpoint);
+  auto acceptor = try_bind(ioctx);
   acceptor.listen();
 
   std::thread server{[&] {
@@ -54,11 +84,8 @@ TEST(HTTPManager, ReadTruncated)
 TEST(HTTPManager, Head)
 {
   using tcp = boost::asio::ip::tcp;
-  tcp::endpoint endpoint(tcp::v4(), 0);
   boost::asio::io_context ioctx;
-  tcp::acceptor acceptor(ioctx);
-  acceptor.open(endpoint.protocol());
-  acceptor.bind(endpoint);
+  auto acceptor = try_bind(ioctx);
   acceptor.listen();
 
   std::thread server{[&] {
-- 
2.39.5