win32_build.sh: mingw-llvm support

winpthreads is a library that emulates the pthreads API using
Windows primitives. It's also used by the mingw/gcc libstdc++
for std::thread.

The issue is that winpthreads isn't well maintained. There
have been numerous bugs that haven't been addressed in years.
Specifically, we've been hitting deadlocks because of the
winpthreads rw lock implementation.

This change will allow building Ceph for Windows using mingw/llvm,
which uses libc++ and doesn't rely on winpthreads.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>

diff --git a/README.windows.rst b/README.windows.rst
index 2c2419c08f7f647300a7579001097910d1e8e3ff..b8065b6b282ba739341ebf70a6da81924a4b9dd7 100644 (file)
--- a/README.windows.rst
+++ b/README.windows.rst
@@ -29,6 +29,8 @@ Flag               Description                      Default value
 =================  ===============================  ===============================
 OS                 Host OS distribution, for mingw  ubuntu (also valid: suse)
                    and other OS specific settings.
+TOOLCHAIN          Mingw toolchain: mingw-llvm or   mingw-llvm
+                   mingw-gcc.
 CEPH_DIR           The Ceph source code directory.  The same as the script.
 BUILD_DIR          The directory where the          $CEPH_DIR/build
                    generated artifacts will be

diff --git a/mingw_conf.sh b/mingw_conf.sh
index aa1ac941849bc5ff170c07af7736ae873938c7a4..7c0d527e1e2d74d3ddbedc29cb979051127b7aaa 100644 (file)
--- a/mingw_conf.sh
+++ b/mingw_conf.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # MINGW Settings:
 # Due to inconsistencies between distributions, mingw versions, binaries,
 # and directories must be determined (or defined) prior to building.
@@ -8,6 +10,16 @@
 # * MINGW_CMAKE_FILE - if set, a cmake toolchain file will be created
 # * MINGW_POSIX_FLAGS - if set, Mingw Posix compatibility mode will be
 #                       enabled by defining the according flags.
+# * USE_MINGW_LLVM - allows using the mingw llvm toolchain
+# * MINGW_LLVM_DIR - allows specifying the mingw-llvm toolchain location.
+#                    If unset, we'll use the default path from build.deps.
+
+SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
+SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    MINGW_LLVM_DIR=${MINGW_LLVM_DIR:-"$SCRIPT_DIR/build.deps/mingw-llvm"}
+fi
 
 # -Common mingw settings-
 MINGW_PREFIX="x86_64-w64-mingw32-"
@@ -17,60 +29,82 @@ MINGW_DLLTOOL="${MINGW_BASE}-dlltool"
 MINGW_WINDRES="${MINGW_BASE}-windres"
 MINGW_STRIP="${MINGW_BASE}-strip"
 MINGW_OBJCOPY="${MINGW_BASE}-objcopy"
-# -Distribution specific mingw settings-
-case "$OS" in
-    ubuntu)
-        mingwPosix="-posix"
-        mingwLibDir="/usr/lib/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
-        mingwLibpthreadDir="/usr/${MINGW_BASE}/lib"
-        PTW32Include=/usr/share/mingw-w64/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/lib
-       ;;
-    rhel)
-        mingwPosix=""
-        mingwLibDir="/usr/lib64/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
-        mingwLibpthreadDir="$mingwTargetLibDir"
-        PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
-       ;;
-    suse)
-        mingwPosix=""
-        mingwLibDir="/usr/lib64/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
-        mingwLibpthreadDir="$mingwTargetLibDir"
-        PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
-        ;;
-    *)
-        echo "$ID is unknown, automatic mingw configuration is not possible."
-        exit 1
-        ;;
-esac
-# -Common mingw settings, dependent upon distribution specific settings-
-MINGW_FIND_ROOT_LIB_PATH="${mingwLibDir}/\${TOOLCHAIN_PREFIX}/${mingwVersion}"
-MINGW_CC="${MINGW_BASE}-gcc${mingwPosix}"
-MINGW_CXX="${MINGW_BASE}-g++${mingwPosix}"
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    # This package isn't currently provided by Linux distributions, we're
+    # fetching it from Github.
+    export PATH="$MINGW_LLVM_DIR/bin:$PATH"
+    mingwPosix=""
+    mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+    mingwX64IncludeDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/include"
+    mingwX64BinDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/bin"
+    mingwX64LibDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/lib"
+    mingwTargetLibDir="$mingwX64BinDir"
+    mingwLibpthreadDir="$mingwX64BinDir"
+    PTW32Include="$mingwX64IncludeDir"
+    PTW32Lib="$mingwX64LibDir"
+
+    MINGW_CC="${MINGW_BASE}-clang${mingwPosix}"
+    MINGW_CXX="${MINGW_BASE}-clang++${mingwPosix}"
+
+    MINGW_FIND_ROOT_PATH="$MINGW_LLVM_DIR/x86_64-w64-mingw32"
+else
+    # -Distribution specific mingw settings-
+    case "$OS" in
+        ubuntu)
+            mingwPosix="-posix"
+            mingwLibDir="/usr/lib/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
+            mingwLibpthreadDir="/usr/${MINGW_BASE}/lib"
+            PTW32Include=/usr/share/mingw-w64/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/lib
+            ;;
+        rhel)
+            mingwPosix=""
+            mingwLibDir="/usr/lib64/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
+            mingwLibpthreadDir="$mingwTargetLibDir"
+            PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
+            ;;
+        suse)
+            mingwPosix=""
+            mingwLibDir="/usr/lib64/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
+            mingwLibpthreadDir="$mingwTargetLibDir"
+            PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
+            ;;
+        *)
+            echo "$ID is unknown, automatic mingw configuration is not possible."
+            exit 1
+            ;;
+    esac
+    MINGW_CC="${MINGW_BASE}-gcc${mingwPosix}"
+    MINGW_CXX="${MINGW_BASE}-g++${mingwPosix}"
+
+    # -Common mingw settings, dependent upon distribution specific settings-
+    MINGW_FIND_ROOT_LIB_PATH="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
+    MINGW_FIND_ROOT_PATH="/usr/${MINGW_BASE} ${MINGW_FIND_ROOT_LIB_PATH}"
+fi
 # End MINGW configuration
 
 
 if [[ -n $MINGW_CMAKE_FILE ]]; then
     cat > $MINGW_CMAKE_FILE <<EOL
 set(CMAKE_SYSTEM_NAME Windows)
-set(TOOLCHAIN_PREFIX ${MINGW_BASE})
 set(CMAKE_SYSTEM_PROCESSOR x86_64)
 
 # We'll need to use posix threads in order to use
 # C++11 features, such as std::thread.
-set(CMAKE_C_COMPILER \${TOOLCHAIN_PREFIX}-gcc${mingwPosix})
-set(CMAKE_CXX_COMPILER \${TOOLCHAIN_PREFIX}-g++${mingwPosix})
-set(CMAKE_RC_COMPILER \${TOOLCHAIN_PREFIX}-windres)
+set(CMAKE_C_COMPILER ${MINGW_CC})
+set(CMAKE_CXX_COMPILER ${MINGW_CXX})
+set(CMAKE_RC_COMPILER ${MINGW_WINDRES})
 
-set(CMAKE_FIND_ROOT_PATH /usr/\${TOOLCHAIN_PREFIX} ${MINGW_FIND_ROOT_LIB_PATH})
+set(CMAKE_FIND_ROOT_PATH ${MINGW_FIND_ROOT_PATH})
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 # TODO: consider switching this to "ONLY". The issue with
 # that is that all our libs should then be under
@@ -88,4 +122,12 @@ add_definitions(-D_POSIX_=1)
 add_definitions(-D_POSIX_THREADS=1)
 EOL
     fi
-fi
\ No newline at end of file
+
+    if [[ -n $USE_MINGW_LLVM ]]; then
+        cat >> $MINGW_CMAKE_FILE <<EOL
+add_definitions(-I$mingwX64IncludeDir)
+add_definitions(-march=native)
+add_definitions(-Wno-unknown-attributes)
+EOL
+    fi
+fi

diff --git a/win32_build.sh b/win32_build.sh
index c29b58491725580eac0fa41c827b948c0caccfc2..b7b395df1929feb57232d45ee6fdfd8398569f10 100755 (executable)
--- a/win32_build.sh
+++ b/win32_build.sh
@@ -60,6 +60,25 @@ if [[ -z $OS ]]; then
 fi
 export OS="$OS"
 
+# The main advantages of mingw-llvm:
+# * not affected by the libstdc++/winpthread rw lock bugs
+# * can generate pdb debug symbols, which are compatible with WinDBG
+TOOLCHAIN=${TOOLCHAIN:-"mingw-llvm"}
+
+case "$TOOLCHAIN" in
+    mingw-llvm)
+        echo "Using mingw-llvm."
+        export USE_MINGW_LLVM=1
+        ;;
+    mingw-gcc)
+        echo "Using mingw-gcc"
+        ;;
+    *)
+        echo "Unsupported toolchain: $TOOLCHAIN."
+        echo "Allowed toolchains: mingw-llvm or mingw-gcc."
+esac
+
+
 # We'll have to be explicit here since auto-detecting doesn't work
 # properly when cross compiling.
 ALLOCATOR=${ALLOCATOR:-libc}
@@ -212,11 +231,18 @@ if [[ -z $SKIP_DLL_COPY ]]; then
         $lz4Dir/lib/dll/liblz4-1.dll
         $sslDir/bin/libcrypto-1_1-x64.dll
         $sslDir/bin/libssl-1_1-x64.dll
-        $mingwTargetLibDir/libstdc++-6.dll
-        $mingwTargetLibDir/libgcc_s_seh-1.dll
-        $mingwTargetLibDir/libssp*.dll
         $mingwLibpthreadDir/libwinpthread-1.dll
         $boostDir/lib/*.dll)
+    if [[ -n $USE_MINGW_LLVM ]]; then
+        required_dlls+=(
+            $mingwTargetLibDir/libc++.dll
+            $mingwTargetLibDir/libunwind.dll)
+    else
+        required_dlls+=(
+            $mingwTargetLibDir/libstdc++-6.dll
+            $mingwTargetLibDir/libssp*.dll
+            $mingwTargetLibDir/libgcc_s_seh-1.dll)
+    fi
     echo "Copying required dlls to $binDir."
     cp ${required_dlls[@]} $binDir
 fi

diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index 980c58c5be9af424c904410c8a1f4193347f4dfa..351cb65e07180139b7abb890986bc03bdd855df2 100755 (executable)
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -5,6 +5,8 @@ set -e
 SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
 SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
 
+USE_MINGW_LLVM=${USE_MINGW_LLVM:-}
+
 num_vcpus=$(nproc)
 NUM_WORKERS=${NUM_WORKERS:-$num_vcpus}
 
@@ -21,6 +23,7 @@ sslSrcDir="${depsSrcDir}/openssl"
 
 # For now, we'll keep the version number within the file path when not using git.
 boostUrl="https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+boostSha256Sum="66a469b6e608a51f8347236f4912e27dc5c60c60d7d53ae9bfe4683316c6f04c"
 boostSrcDir="${depsSrcDir}/boost_1_82_0"
 boostDir="${depsToolsetDir}/boost"
 zlibDir="${depsToolsetDir}/zlib"
@@ -43,6 +46,10 @@ dokanTag="v2.0.5.1000"
 dokanSrcDir="${depsSrcDir}/dokany"
 dokanLibDir="${depsToolsetDir}/dokany/lib"
 
+mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-msvcrt-ubuntu-18.04-x86_64.tar.xz"
+mingwLlvmSha256Sum="bc97745e702fb9e8f2a16f7d09dd5061ceeef16554dd12e542f619ce937e8d7a"
+mingwLlvmDir="${DEPS_DIR}/mingw-llvm"
+
 # Allow for OS specific customizations through the OS flag (normally
 # passed through from win32_build).
 # Valid options are currently "ubuntu", "rhel", and "suse".
@@ -100,6 +107,22 @@ case "$OS" in
         ;;
 esac
 
+if [[ -n $USE_MINGW_LLVM && ! -d $mingwLlvmDir ]]; then
+    echo "Fetching mingw-llvm"
+    cd $DEPS_DIR
+    wget -q -O mingw-llvm.tar.xz $mingwLlvmUrl
+    checksum=`sha256sum mingw-llvm.tar.xz | cut -d ' ' -f 1`
+    if [[ "$mingwLlvmSha256Sum" != "$checksum" ]]; then
+        echo "Invalid mingw-llvm checksum: $checksum" >&2
+        exit 1
+    fi
+    tar xJf mingw-llvm.tar.xz
+    rm mingw-llvm.tar.xz
+    # Remove the version from the mingw-llvm dirname, making it easier to locate
+    # and avoiding MAX_PATH issues with WSL.
+    mv `basename $mingwLlvmUrl | sed 's/\.tar\..*//g'` $mingwLlvmDir
+fi
+
 MINGW_CMAKE_FILE="$DEPS_DIR/mingw.cmake"
 source "$SCRIPT_DIR/mingw_conf.sh"
 
@@ -148,17 +171,33 @@ echo "Building boost."
 cd $depsSrcDir
 if [[ ! -d $boostSrcDir ]]; then
     echo "Downloading boost."
-    wget -qO- $boostUrl | tar xz
+    wget -q -O boost.tar.gz $boostUrl
+    checksum=`sha256sum boost.tar.gz | cut -d ' ' -f 1`
+    if [[ "$boostSha256Sum" != "$checksum" ]]; then
+        echo "Invalid boost checksum: $checksum" >&2
+        exit 1
+    fi
+    tar xzf boost.tar.gz
+    rm boost.tar.gz
 fi
 
 cd $boostSrcDir
-echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    b2toolset="clang"
+    echo "using clang :  : ${MINGW_CXX} ;" > user-config.jam
+else
+    b2toolset="gcc-mingw32"
+    echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
+fi
 
 # Workaround for https://github.com/boostorg/thread/issues/156
 # Older versions of mingw provided a different pthread lib.
 sed -i 's/lib$(libname)GC2.a/lib$(libname).a/g' ./libs/thread/build/Jamfile.v2
-sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
-sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
+if [[ -z $USE_MINGW_LLVM ]]; then
+    sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
+    sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
+fi
 
 export PTW32_INCLUDE=${PTW32Include}
 export PTW32_LIB=${PTW32Lib}
@@ -197,7 +236,7 @@ EOL
 
 ./bootstrap.sh
 
-./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
+./b2 install --user-config=user-config.jam toolset=$b2toolset \
     target-os=windows release \
     link=static,shared \
     threadapi=win32 --prefix=$boostDir \