From 35b67b2558fd4b7c39ca10e5b13b4dc6c983ce3c Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Mon, 4 Jul 2022 13:59:46 +0000
Subject: [PATCH] common: use boost::shared_mutex on Windows

The winpthreads shared mutex implementation causes deadlocks on
Windows [1][2]. Specifically, async RBD IO calls are hanging. This
also prevents the images from being unmounted.

For this reason, we're switching to boost::shared_mutex when using
MinGW.

[1] https://github.com/cloudbase/wnbd/issues/63#issuecomment-1161547102
[2] https://github.com/msys2/MINGW-packages/issues/3319
Trace: https://pastebin.com/raw/i3jpTyS3

Fixes: https://tracker.ceph.com/issues/56480
Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 CMakeLists.txt          | 10 +++++++++-
 src/common/ceph_mutex.h | 23 +++++++++++++++++++----
 win32_deps_build.sh     |  2 +-
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c0870323a94..de7160055f1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,7 +39,15 @@ if(WIN32)
   # the targeted Windows version. The availability of certain functions and
   # structures will depend on it.
   set(WIN32_WINNT "0x0A00" CACHE STRING "Targeted Windows version.")
-  add_definitions(-D_WIN32_WINNT=${WIN32_WINNT})
+  # In order to avoid known winpthread issues, we're using the boost
+  # shared mutex implementation.
+  # https://github.com/msys2/MINGW-packages/issues/3319
+  add_definitions(
+    -D_WIN32_WINNT=${WIN32_WINNT}
+    -DBOOST_THREAD_PROVIDES_GENERIC_SHARED_MUTEX_ON_WIN
+    -DBOOST_THREAD_V2_SHARED_MUTEX
+  )
+  set(Boost_THREADAPI "win32")
 endif()
 
 if(MINGW)
diff --git a/src/common/ceph_mutex.h b/src/common/ceph_mutex.h
index 4029a5a9f15..81777c7dbe2 100644
--- a/src/common/ceph_mutex.h
+++ b/src/common/ceph_mutex.h
@@ -69,6 +69,9 @@ namespace ceph {
 // ============================================================================
 // debug (lockdep-capable, various sanity checks and asserts)
 // ============================================================================
+//
+// Note: this is known to cause deadlocks on Windows because
+// of the winpthreads shared mutex implementation.
 
 #include "common/condition_variable_debug.h"
 #include "common/mutex_debug.h"
@@ -115,27 +118,39 @@ namespace ceph {
 
 #include <condition_variable>
 #include <mutex>
-#include <shared_mutex>
 
+// The winpthreads shared mutex implementation is broken.
+// We'll use boost::shared_mutex instead.
+// https://github.com/msys2/MINGW-packages/issues/3319
+#if __MINGW32__
+#include <boost/thread/shared_mutex.hpp>
+#else
+#include <shared_mutex>
+#endif
 
 namespace ceph {
 
   typedef std::mutex mutex;
   typedef std::recursive_mutex recursive_mutex;
   typedef std::condition_variable condition_variable;
+
+#if __MINGW32__
+  typedef boost::shared_mutex shared_mutex;
+#else
   typedef std::shared_mutex shared_mutex;
+#endif
 
   // discard arguments to make_mutex (they are for debugging only)
   template <typename ...Args>
-  std::mutex make_mutex(Args&& ...args) {
+  mutex make_mutex(Args&& ...args) {
     return {};
   }
   template <typename ...Args>
-  std::recursive_mutex make_recursive_mutex(Args&& ...args) {
+  recursive_mutex make_recursive_mutex(Args&& ...args) {
     return {};
   }
   template <typename ...Args>
-  std::shared_mutex make_shared_mutex(Args&& ...args) {
+  shared_mutex make_shared_mutex(Args&& ...args) {
     return {};
   }
 
diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index dba562dfadf..2baefd98821 100755
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -201,7 +201,7 @@ EOL
 ./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
     target-os=windows release \
     link=static,shared \
-    threadapi=pthread --prefix=$boostDir \
+    threadapi=win32 --prefix=$boostDir \
     address-model=64 architecture=x86 \
     binary-format=pe abi=ms -j $NUM_WORKERS \
     -sZLIB_INCLUDE=$zlibDir/include -sZLIB_LIBRARY_PATH=$zlibDir/lib \
-- 
2.39.5