common: Windows Unicode CLI support

Windows CLI arguments use either ANSI (main()) or UTF-16 (wmain()).
Meanwhile, Ceph libraries expect UTF-8 and raise exceptions when
trying to use Unicode CLI arguments or log Unicode output:

  rbd.exe create test_unicode_șțăâ --size=32M
  terminate called after throwing an instance of 'std::runtime_error'
    what():  invalid utf8

We'll use a Windows application manifest, setting the "activeCodePage"
property [1][2]. This enables the Windows UCRT UTF-8 mode so that
functions that receive char* arguments will expect UTF-8 instead of ANSI,
including main(). One exception is CreateProcess, which will need the
UTF-16 form (CreateProcessW).

Despite the locale being set to utf-8, we'll have to explicitly set
the console output to utf-8 using SetConsoleOutputCP(CP_UTF8).

In order to use the UTF-8 locale, we'll have to switch the mingw-llvm
runtime from msvcrt to ucrt.

This also fixes ceph-dokan crashes that currently occur when non-ANSI
paths are logged.

[1] https://learn.microsoft.com/en-us/windows/win32/sbscs/application-manifests#activecodepage
[2] https://learn.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-code-page

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>

diff --git a/mingw_conf.sh b/mingw_conf.sh
index 6a226da5f040db2fd08741a9e18859c61b45f1ec..a03eb95dda3b81cab85aecce543becc14476755b 100644 (file)
--- a/mingw_conf.sh
+++ b/mingw_conf.sh
@@ -129,8 +129,8 @@ EOL
     if [[ -n $USE_MINGW_LLVM ]]; then
         cat >> $MINGW_CMAKE_FILE <<EOL
 add_definitions(-I$mingwX64IncludeDir)
-add_definitions(-march=native)
-add_definitions(-Wno-unknown-attributes)
+add_compile_options(-march=native)
+add_compile_options(-Wno-unknown-attributes)
 EOL
     fi
 fi

diff --git a/src/common/win32/SubProcess.cc b/src/common/win32/SubProcess.cc
index 3ed3b4f54c711756c4d6c6d23598a84ff293f46e..59975b1e6d318adedb845dafa394cacdda39152a 100644 (file)
--- a/src/common/win32/SubProcess.cc
+++ b/src/common/win32/SubProcess.cc
@@ -18,6 +18,7 @@
 
 #include "common/SubProcess.h"
 #include "common/errno.h"
+#include "common/win32/wstring.h"
 #include "include/ceph_assert.h"
 #include "include/compat.h"
 
@@ -174,8 +175,9 @@ int SubProcess::spawn() {
   for (auto& arg : cmd_args) {
     cmdline << " " << std::quoted(arg);
   }
+  std::wstring cmdline_w = to_wstring(cmdline.str());
 
-  STARTUPINFO si = {0};
+  STARTUPINFOW si = {0};
   PROCESS_INFORMATION pi = {0};
   SECURITY_ATTRIBUTES sa = {0};
 
@@ -224,8 +226,8 @@ int SubProcess::spawn() {
   // We've transfered ownership from those handles.
   stdin_w = stdout_r = stderr_r = INVALID_HANDLE_VALUE;
 
-  if (!CreateProcess(
-      NULL, const_cast<char*>(cmdline.str().c_str()),
+  if (!CreateProcessW(
+      NULL, const_cast<wchar_t*>(cmdline_w.c_str()),
       NULL, NULL, /* No special security attributes */
       1, /* Inherit handles marked as inheritable */
       0, /* No special flags */

diff --git a/src/common/win32/code_page.manifest b/src/common/win32/code_page.manifest
new file mode 100644 (file)
index 0000000..dab929e
--- /dev/null
+++ b/src/common/win32/code_page.manifest
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>

diff --git a/src/common/win32/code_page.rc b/src/common/win32/code_page.rc
new file mode 100644 (file)
index 0000000..12258c4
--- /dev/null
+++ b/src/common/win32/code_page.rc
@@ -0,0 +1,2 @@
+#include <winuser.h>
+CREATEPROCESS_MANIFEST_RESOURCE_ID RT_MANIFEST "code_page.manifest"

diff --git a/src/dokan/CMakeLists.txt b/src/dokan/CMakeLists.txt
index cc05a0f29f6045b2b390ffabec0d94e1d8964040..2a61d38bb8d04d27175f6409328b594249090308 100644 (file)
--- a/src/dokan/CMakeLists.txt
+++ b/src/dokan/CMakeLists.txt
@@ -2,7 +2,8 @@ set(ceph_dokan_srcs
   ceph_dokan.cc
   dbg.cc
   utils.cc
-  options.cc)
+  options.cc
+  ../common/win32/code_page.rc)
 add_executable(ceph-dokan ${ceph_dokan_srcs})
 target_link_libraries(ceph-dokan ${DOKAN_LIBRARIES}
   ${GSSAPI_LIBRARIES}

diff --git a/src/dokan/ceph_dokan.cc b/src/dokan/ceph_dokan.cc
index 9e115222cab2f3def2ecba578521efd485a78c67..1ea82e07423b0a7ac91a0c75605c211f445b36e4 100644 (file)
--- a/src/dokan/ceph_dokan.cc
+++ b/src/dokan/ceph_dokan.cc
@@ -1043,6 +1043,8 @@ boost::intrusive_ptr<CephContext> do_global_init(
 
 int main(int argc, const char** argv)
 {
+  SetConsoleOutputCP(CP_UTF8);
+
   if (!SetConsoleCtrlHandler((PHANDLER_ROUTINE)ConsoleHandler, TRUE)) {
     cerr << "Couldn't initialize console event handler." << std::endl;
     return -EINVAL;

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index d1133798c5c9d4d500faa7e96ffae0b4abca661a..993fadb2e2edcc6c43c5c9dd14f6a25cf158dc18 100644 (file)
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -6,6 +6,9 @@ set(rados_srcs
   ${PROJECT_SOURCE_DIR}/src/common/util.cc
   ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc
   ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc)
+if(WIN32)
+  list(APPEND rados_srcs ../common/win32/code_page.rc)
+endif()
 add_executable(rados ${rados_srcs})
 
 target_link_libraries(rados librados global ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 3d602e749cdee9aec3625c6e3cfec0366b7fb8f9..b8cf5e4d1dc369e0c068ec696ab6d28f805adb71 100644 (file)
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -4047,6 +4047,9 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
 
 int main(int argc, const char **argv)
 {
+  #ifdef _WIN32
+  SetConsoleOutputCP(CP_UTF8);
+  #endif
   auto args = argv_to_vec(argc, argv);
   if (args.empty()) {
     cerr << argv[0] << ": -h or --help for usage" << std::endl;

diff --git a/src/tools/rbd/CMakeLists.txt b/src/tools/rbd/CMakeLists.txt
index 19b4e806a752ff038b44982c8619fe017f97a56f..dac1d8babf92de0f6b2db82d80e5f3918aa7d250 100644 (file)
--- a/src/tools/rbd/CMakeLists.txt
+++ b/src/tools/rbd/CMakeLists.txt
@@ -55,6 +55,9 @@ set(rbd_srcs
   action/Ubbd.cc
   action/Watch.cc
   action/Wnbd.cc)
+if(WIN32)
+  list(APPEND rbd_srcs ../../common/win32/code_page.rc)
+endif()
 
 add_executable(rbd ${rbd_srcs}
   $<TARGET_OBJECTS:common_texttable_obj>)

diff --git a/src/tools/rbd/rbd.cc b/src/tools/rbd/rbd.cc
index a8c59d57577b7ebe6dd13063d68aea49fe86482c..bdeded4a05b5db9865a2bd1414bda9d51f4515aa 100644 (file)
--- a/src/tools/rbd/rbd.cc
+++ b/src/tools/rbd/rbd.cc
@@ -5,6 +5,9 @@
 
 int main(int argc, const char **argv)
 {
+  #ifdef _WIN32
+  SetConsoleOutputCP(CP_UTF8);
+  #endif
   rbd::Shell shell;
   return shell.execute(argc, argv);
 }

diff --git a/src/tools/rbd_wnbd/CMakeLists.txt b/src/tools/rbd_wnbd/CMakeLists.txt
index 86c41b2eeb6f13ab88590c4e27913683d5bf3a7b..ff09cd80a15267ad7af5fb0dfc0e42fda5fa880a 100644 (file)
--- a/src/tools/rbd_wnbd/CMakeLists.txt
+++ b/src/tools/rbd_wnbd/CMakeLists.txt
@@ -1,4 +1,7 @@
-add_executable(rbd-wnbd rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc)
+add_executable(
+    rbd-wnbd
+    rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc
+    ../../common/win32/code_page.rc)
 set_target_properties(
     rbd-wnbd PROPERTIES COMPILE_FLAGS
     "-fpermissive -I${WNBD_INCLUDE_DIRS}")

diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc
index d2df88cabb48b428cff54d3a1f4688a228edd36a..1946e83ff967c9858deb0f6be19f0d41bbda6911 100644 (file)
--- a/src/tools/rbd_wnbd/rbd_wnbd.cc
+++ b/src/tools/rbd_wnbd/rbd_wnbd.cc
@@ -331,7 +331,7 @@ int send_map_request(std::string arguments) {
 // which will allow it to communicate the mapping status
 int map_device_using_suprocess(std::string arguments, int timeout_ms)
 {
-  STARTUPINFO si;
+  STARTUPINFOW si;
   PROCESS_INFORMATION pi;
   char ch;
   DWORD err = 0, status = 0;
@@ -407,11 +407,12 @@ int map_device_using_suprocess(std::string arguments, int timeout_ms)
 
   dout(5) << __func__ << ": command line: " << command_line.str() << dendl;
 
-  GetStartupInfo(&si);
+  GetStartupInfoW(&si);
   // Create a detached child
-  if (!CreateProcess(NULL, (char*)command_line.str().c_str(),
-                     NULL, NULL, FALSE, DETACHED_PROCESS,
-                     NULL, NULL, &si, &pi)) {
+  if (!CreateProcessW(
+      NULL, const_cast<wchar_t*>(to_wstring(command_line.str()).c_str()),
+      NULL, NULL, FALSE, DETACHED_PROCESS,
+      NULL, NULL, &si, &pi)) {
     err = GetLastError();
     derr << "CreateProcess failed: " << win32_strerror(err) << dendl;
     exit_code = -ECHILD;
@@ -1904,6 +1905,8 @@ int main(int argc, const char *argv[])
   SetConsoleCtrlHandler(console_handler_routine, true);
   // Avoid the Windows Error Reporting dialog.
   SetErrorMode(GetErrorMode() | SEM_NOGPFAULTERRORBOX);
+  SetConsoleOutputCP(CP_UTF8);
+
   int r = rbd_wnbd(argc, argv);
   if (r < 0) {
     return r;

diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index 6eea81d1b8a212b468ddc1b8481c59c0b0680aa5..c61cd7cb78a8184db10d00980ed48d03a976336f 100755 (executable)
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -40,8 +40,8 @@ dokanTag="v2.0.5.1000"
 dokanSrcDir="${depsSrcDir}/dokany"
 dokanLibDir="${depsToolsetDir}/dokany/lib"
 
-mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-msvcrt-ubuntu-18.04-x86_64.tar.xz"
-mingwLlvmSha256Sum="bc97745e702fb9e8f2a16f7d09dd5061ceeef16554dd12e542f619ce937e8d7a"
+mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-ubuntu-18.04-x86_64.tar.xz"
+mingwLlvmSha256Sum="bc367753dea829d219be32e2e64e2d15d03158ce8e700ae5210ca3d78e6a07ea"
 mingwLlvmDir="${DEPS_DIR}/mingw-llvm"
 
 function _make() {