From 96e8850ff3b6661c86eb361051282c8d66ca2031 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Mon, 12 Jun 2023 08:00:40 +0000 Subject: [PATCH] rbd-wnbd: introduce RbdMapping class We're moving most of the WNBD mapping handling to a separate class called RbdMapping. This simplifies cleanup and makes it easier to reuse. The WnbdHandler class covers WNBD specific operations and IO callbacks while the RbdMapping wrapper will take care of RBD operations. A subsequent change will make use of it while switching from one process per mapping to a single process per host. While at it, we're also moving the rbd-wnbd config helpers to separate files. Signed-off-by: Lucian Petrut --- src/tools/rbd_wnbd/CMakeLists.txt | 1 + src/tools/rbd_wnbd/rbd_mapping.cc | 180 ++++++++++++++ src/tools/rbd_wnbd/rbd_mapping.h | 88 +++++++ src/tools/rbd_wnbd/rbd_mapping_config.cc | 119 +++++++++ src/tools/rbd_wnbd/rbd_mapping_config.h | 77 ++++++ src/tools/rbd_wnbd/rbd_wnbd.cc | 294 +---------------------- src/tools/rbd_wnbd/rbd_wnbd.h | 61 +---- src/tools/rbd_wnbd/wnbd_handler.h | 2 - 8 files changed, 479 insertions(+), 343 deletions(-) create mode 100644 src/tools/rbd_wnbd/rbd_mapping.cc create mode 100644 src/tools/rbd_wnbd/rbd_mapping.h create mode 100644 src/tools/rbd_wnbd/rbd_mapping_config.cc create mode 100644 src/tools/rbd_wnbd/rbd_mapping_config.h diff --git a/src/tools/rbd_wnbd/CMakeLists.txt b/src/tools/rbd_wnbd/CMakeLists.txt index ff09cd80a1526..e757c71601eb7 100644 --- a/src/tools/rbd_wnbd/CMakeLists.txt +++ b/src/tools/rbd_wnbd/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable( rbd-wnbd + rbd_mapping.cc rbd_mapping_config.cc rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc ../../common/win32/code_page.rc) set_target_properties( diff --git a/src/tools/rbd_wnbd/rbd_mapping.cc b/src/tools/rbd_wnbd/rbd_mapping.cc new file mode 100644 index 0000000000000..db136dbf6b92c --- /dev/null +++ b/src/tools/rbd_wnbd/rbd_mapping.cc @@ -0,0 +1,180 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2023 Cloudbase Solutions + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "rbd_mapping.h" + +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" + +#include "global/global_init.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "rbd-wnbd: " + + +int RbdMapping::init() +{ + librbd::image_info_t info; + + int r = rados.ioctx_create(cfg.poolname.c_str(), io_ctx); + if (r < 0) { + derr << "rbd-wnbd: couldn't create IO context: " << cpp_strerror(r) + << dendl; + return r; + } + + io_ctx.set_namespace(cfg.nsname); + + r = rbd.open(io_ctx, image, cfg.imgname.c_str()); + if (r < 0) { + derr << "rbd-wnbd: couldn't open rbd image: " << cpp_strerror(r) + << dendl; + return r; + } + + if (cfg.exclusive) { + r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE); + if (r < 0) { + derr << "rbd-wnbd: failed to acquire exclusive lock: " << cpp_strerror(r) + << dendl; + return r; + } + } + + if (!cfg.snapname.empty()) { + r = image.snap_set(cfg.snapname.c_str()); + if (r < 0) { + derr << "rbd-wnbd: couldn't use snapshot: " << cpp_strerror(r) + << dendl; + return r; + } + } + + r = image.stat(info, sizeof(info)); + if (r < 0) + return r; + + initial_image_size = info.size; + + // We're storing mapping details in the registry even for non-persistent + // mappings. This allows us to easily retrieve mapping details such + // as the rbd pool or admin socket path. + // We're cleaning up the registry entry when the non-persistent mapping + // gets disconnected or when the ceph service restarts. + r = save_config_to_registry(&cfg, command_line); + if (r < 0) + return r; + + handler = new WnbdHandler(image, cfg.devpath, + info.size / RBD_WNBD_BLKSIZE, + RBD_WNBD_BLKSIZE, + !cfg.snapname.empty() || cfg.readonly, + g_conf().get_val("rbd_cache"), + cfg.io_req_workers, + cfg.io_reply_workers); + return 0; +} + +void RbdMapping::shutdown() +{ + std::unique_lock l{shutdown_lock}; + + int r = 0; + if (!cfg.persistent) { + dout(5) << __func__ << ": cleaning up non-persistent mapping: " + << cfg.devpath << dendl; + r = remove_config_from_registry(&cfg); + if (r) { + derr << __func__ << ": could not clean up non-persistent mapping: " + << cfg.devpath << ". Error: " << cpp_strerror(r) << dendl; + } + } + + if (watch_ctx) { + r = image.update_unwatch(watch_handle); + if (r < 0) { + derr << __func__ << ": update_unwatch failed with error: " + << cpp_strerror(r) << dendl; + } + delete watch_ctx; + watch_ctx = nullptr; + } + + if (handler) { + handler->shutdown(); + delete handler; + handler = nullptr; + } + + image.close(); + io_ctx.close(); + rados.shutdown(); +} + +int RbdMapping::start() +{ + int r = init(); + if (r < 0) { + return r; + } + + r = handler->start(); + if (r) { + return r == ERROR_ALREADY_EXISTS ? -EEXIST : -EINVAL; + } + + watch_ctx = new WNBDWatchCtx(io_ctx, handler, image, initial_image_size); + r = image.update_watch(watch_ctx, &watch_handle); + if (r < 0) { + derr << __func__ << ": update_watch failed with error: " + << cpp_strerror(r) << dendl; + return r; + } + + // We're informing the parent processes that the initialization + // was successful. + int err = 0; + if (!cfg.parent_pipe.empty()) { + HANDLE parent_pipe_handle = CreateFile( + cfg.parent_pipe.c_str(), GENERIC_WRITE, 0, NULL, + OPEN_EXISTING, 0, NULL); + if (parent_pipe_handle == INVALID_HANDLE_VALUE) { + err = GetLastError(); + derr << "Could not open parent pipe: " << win32_strerror(err) << dendl; + } else if (!WriteFile(parent_pipe_handle, "a", 1, NULL, NULL)) { + // TODO: consider exiting in this case. The parent didn't wait for us, + // maybe it was killed after a timeout. + err = GetLastError(); + derr << "Failed to communicate with the parent: " + << win32_strerror(err) << dendl; + } else { + dout(5) << __func__ << ": submitted parent notification." << dendl; + } + + if (parent_pipe_handle != INVALID_HANDLE_VALUE) + CloseHandle(parent_pipe_handle); + + global_init_postfork_finish(g_ceph_context); + } + + return 0; +} + +int RbdMapping::wait() { + if (handler) { + return handler->wait(); + } + return 0; +} diff --git a/src/tools/rbd_wnbd/rbd_mapping.h b/src/tools/rbd_wnbd/rbd_mapping.h new file mode 100644 index 0000000000000..ce96beeec9e30 --- /dev/null +++ b/src/tools/rbd_wnbd/rbd_mapping.h @@ -0,0 +1,88 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2023 Cloudbase Solutions + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include "rbd_mapping_config.h" +#include "wnbd_handler.h" + +class WNBDWatchCtx : public librbd::UpdateWatchCtx +{ +private: + librados::IoCtx &io_ctx; + WnbdHandler* handler; + librbd::Image ℑ + uint64_t size; +public: + WNBDWatchCtx(librados::IoCtx& io_ctx, WnbdHandler* handler, + librbd::Image& image, uint64_t size) + : io_ctx(io_ctx) + , handler(handler) + , image(image) + , size(size) + { + } + + ~WNBDWatchCtx() override {} + + void handle_notify() override + { + uint64_t new_size; + + if (image.size(&new_size) == 0 && new_size != size && + handler->resize(new_size) == 0) { + size = new_size; + } + } +}; + + +class RbdMapping +{ +private: + Config cfg; + // We're sharing the rados object across mappings in order to + // reuse the OSD connections. + librados::Rados &rados; + std::string command_line; + + librbd::RBD rbd; + librados::IoCtx io_ctx; + librbd::Image image; + uint64_t initial_image_size; + + WnbdHandler* handler = nullptr; + uint64_t watch_handle; + WNBDWatchCtx* watch_ctx = nullptr; + + ceph::mutex shutdown_lock = ceph::make_mutex("RbdMapping::ShutdownLock"); + + int init(); + void shutdown(); + +public: + RbdMapping(Config& _cfg, librados::Rados& _rados, + std::string _command_line) + : cfg(_cfg) + , rados(_rados) + , command_line(_command_line) + { + } + + ~RbdMapping() + { + shutdown(); + } + + int start(); + int wait(); +}; diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.cc b/src/tools/rbd_wnbd/rbd_mapping_config.cc new file mode 100644 index 0000000000000..c78920ea8d32a --- /dev/null +++ b/src/tools/rbd_wnbd/rbd_mapping_config.cc @@ -0,0 +1,119 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 SUSE LINUX GmbH + * Copyright (C) 2023 Cloudbase Solutions + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "rbd_mapping_config.h" + +#include "common/debug.h" +#include "common/dout.h" +#include "common/win32/registry.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "rbd-wnbd: " + +int construct_devpath_if_missing(Config* cfg) +{ + // Windows doesn't allow us to request specific disk paths when mapping an + // image. This will just be used by rbd-wnbd and wnbd as an identifier. + if (cfg->devpath.empty()) { + if (cfg->imgname.empty()) { + derr << "Missing image name." << dendl; + return -EINVAL; + } + + if (!cfg->poolname.empty()) { + cfg->devpath += cfg->poolname; + cfg->devpath += '/'; + } + if (!cfg->nsname.empty()) { + cfg->devpath += cfg->nsname; + cfg->devpath += '/'; + } + + cfg->devpath += cfg->imgname; + + if (!cfg->snapname.empty()) { + cfg->devpath += '@'; + cfg->devpath += cfg->snapname; + } + } + + return 0; +} + +int save_config_to_registry(Config* cfg, std::string command_line) +{ + std::string strKey{ SERVICE_REG_KEY }; + strKey.append("\\"); + strKey.append(cfg->devpath); + auto reg_key = RegistryKey( + g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), true); + if (!reg_key.hKey) { + return -EINVAL; + } + + int ret_val = 0; + // Registry writes are immediately available to other processes. + // Still, we'll do a flush to ensure that the mapping can be + // recreated after a system crash. + if (reg_key.set("pid", getpid()) || + reg_key.set("devpath", cfg->devpath) || + reg_key.set("poolname", cfg->poolname) || + reg_key.set("nsname", cfg->nsname) || + reg_key.set("imgname", cfg->imgname) || + reg_key.set("snapname", cfg->snapname) || + reg_key.set("command_line", command_line) || + reg_key.set("persistent", cfg->persistent) || + reg_key.set("admin_sock_path", g_conf()->admin_socket) || + reg_key.flush()) { + ret_val = -EINVAL; + } + + return ret_val; +} + +int remove_config_from_registry(Config* cfg) +{ + std::string strKey{ SERVICE_REG_KEY }; + strKey.append("\\"); + strKey.append(cfg->devpath); + return RegistryKey::remove( + g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str()); +} + +int load_mapping_config_from_registry(std::string devpath, Config* cfg) +{ + std::string strKey{ SERVICE_REG_KEY }; + strKey.append("\\"); + strKey.append(devpath); + auto reg_key = RegistryKey( + g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), false); + if (!reg_key.hKey) { + if (reg_key.missingKey) + return -ENOENT; + else + return -EINVAL; + } + + reg_key.get("devpath", cfg->devpath); + reg_key.get("poolname", cfg->poolname); + reg_key.get("nsname", cfg->nsname); + reg_key.get("imgname", cfg->imgname); + reg_key.get("snapname", cfg->snapname); + reg_key.get("command_line", cfg->command_line); + reg_key.get("persistent", cfg->persistent); + reg_key.get("admin_sock_path", cfg->admin_sock_path); + + return 0; +} diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.h b/src/tools/rbd_wnbd/rbd_mapping_config.h new file mode 100644 index 0000000000000..60e3fa20a45e4 --- /dev/null +++ b/src/tools/rbd_wnbd/rbd_mapping_config.h @@ -0,0 +1,77 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 SUSE LINUX GmbH + * Copyright (C) 2023 Cloudbase Solutions + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include + +#include + +#define SERVICE_REG_KEY "SYSTEM\\CurrentControlSet\\Services\\rbd-wnbd" + +#define DEFAULT_SERVICE_START_TIMEOUT 120 +#define DEFAULT_IMAGE_MAP_TIMEOUT 20 +#define DEFAULT_SERVICE_THREAD_COUNT 8 +#define DEFAULT_SOFT_REMOVE_TIMEOUT 15 +#define DEFAULT_IO_WORKER_COUNT 4 + +#define RBD_WNBD_BLKSIZE 512UL + +struct Config { + bool exclusive = false; + bool readonly = false; + + std::string parent_pipe; + + std::string poolname; + std::string nsname; + std::string imgname; + std::string snapname; + std::string devpath; + + std::string format; + bool pretty_format = false; + + bool hard_disconnect = false; + int soft_disconnect_timeout = DEFAULT_SOFT_REMOVE_TIMEOUT; + bool hard_disconnect_fallback = true; + + int service_start_timeout = DEFAULT_SERVICE_START_TIMEOUT; + int image_map_timeout = DEFAULT_IMAGE_MAP_TIMEOUT; + bool remap_failure_fatal = false; + bool adapter_monitoring_enabled = false; + + // TODO: consider moving those fields to a separate structure. Those + // provide connection information without actually being configurable. + // The disk number is provided by Windows. + int disk_number = -1; + int pid = 0; + std::string serial_number; + bool active = false; + bool wnbd_mapped = false; + std::string command_line; + std::string admin_sock_path; + + WnbdLogLevel wnbd_log_level = WnbdLogLevelInfo; + int io_req_workers = DEFAULT_IO_WORKER_COUNT; + int io_reply_workers = DEFAULT_IO_WORKER_COUNT; + int service_thread_count = DEFAULT_SERVICE_THREAD_COUNT; + + // register the mapping, recreating it when the Ceph service starts. + bool persistent = true; +}; + +int construct_devpath_if_missing(Config* cfg); +int save_config_to_registry(Config* cfg, std::string command_line); +int remove_config_from_registry(Config* cfg); +int load_mapping_config_from_registry(std::string devpath, Config* cfg); diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc index 1946e83ff967c..08cb7ca9359d1 100644 --- a/src/tools/rbd_wnbd/rbd_wnbd.cc +++ b/src/tools/rbd_wnbd/rbd_wnbd.cc @@ -307,7 +307,7 @@ int send_map_request(std::string arguments) { &reply, sizeof(reply), &bytes_read, - DEFAULT_MAP_TIMEOUT_MS); + DEFAULT_IMAGE_MAP_TIMEOUT * 1000); if (!success) { DWORD err = GetLastError(); derr << "Could not send device map request. " @@ -519,79 +519,11 @@ BOOL WINAPI console_handler_routine(DWORD dwCtrlType) dout(0) << "Received control signal: " << dwCtrlType << ". Exiting." << dendl; - std::unique_lock l{shutdown_lock}; - if (handler) - handler->shutdown(); + // TODO: shutdown all mappings return true; } -int save_config_to_registry(Config* cfg) -{ - std::string strKey{ SERVICE_REG_KEY }; - strKey.append("\\"); - strKey.append(cfg->devpath); - auto reg_key = RegistryKey( - g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), true); - if (!reg_key.hKey) { - return -EINVAL; - } - - int ret_val = 0; - // Registry writes are immediately available to other processes. - // Still, we'll do a flush to ensure that the mapping can be - // recreated after a system crash. - if (reg_key.set("pid", getpid()) || - reg_key.set("devpath", cfg->devpath) || - reg_key.set("poolname", cfg->poolname) || - reg_key.set("nsname", cfg->nsname) || - reg_key.set("imgname", cfg->imgname) || - reg_key.set("snapname", cfg->snapname) || - reg_key.set("command_line", get_cli_args()) || - reg_key.set("persistent", cfg->persistent) || - reg_key.set("admin_sock_path", g_conf()->admin_socket) || - reg_key.flush()) { - ret_val = -EINVAL; - } - - return ret_val; -} - -int remove_config_from_registry(Config* cfg) -{ - std::string strKey{ SERVICE_REG_KEY }; - strKey.append("\\"); - strKey.append(cfg->devpath); - return RegistryKey::remove( - g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str()); -} - -int load_mapping_config_from_registry(string devpath, Config* cfg) -{ - std::string strKey{ SERVICE_REG_KEY }; - strKey.append("\\"); - strKey.append(devpath); - auto reg_key = RegistryKey( - g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), false); - if (!reg_key.hKey) { - if (reg_key.missingKey) - return -ENOENT; - else - return -EINVAL; - } - - reg_key.get("devpath", cfg->devpath); - reg_key.get("poolname", cfg->poolname); - reg_key.get("nsname", cfg->nsname); - reg_key.get("imgname", cfg->imgname); - reg_key.get("snapname", cfg->snapname); - reg_key.get("command_line", cfg->command_line); - reg_key.get("persistent", cfg->persistent); - reg_key.get("admin_sock_path", cfg->admin_sock_path); - - return 0; -} - int restart_registered_mappings( int worker_count, int total_timeout, @@ -793,7 +725,8 @@ class RBDService : public ServiceBase { // TODO: use the configured service map timeout. // TODO: add ceph.conf options. return map_device_using_suprocess( - (char*)request->arguments, DEFAULT_MAP_TIMEOUT_MS); + (char*)request->arguments, + DEFAULT_IMAGE_MAP_TIMEOUT * 1000); default: dout(1) << "Received unsupported command: " << request->command << dendl; @@ -1042,35 +975,6 @@ exit: } }; -class WNBDWatchCtx : public librbd::UpdateWatchCtx -{ -private: - librados::IoCtx &io_ctx; - WnbdHandler* handler; - librbd::Image ℑ - uint64_t size; -public: - WNBDWatchCtx(librados::IoCtx& io_ctx, WnbdHandler* handler, - librbd::Image& image, uint64_t size) - : io_ctx(io_ctx) - , handler(handler) - , image(image) - , size(size) - { } - - ~WNBDWatchCtx() override {} - - void handle_notify() override - { - uint64_t new_size; - - if (image.size(&new_size) == 0 && new_size != size && - handler->resize(new_size) == 0) { - size = new_size; - } - } -}; - static void usage() { const char* usage_str =R"( @@ -1138,36 +1042,6 @@ Common options: static Command cmd = None; -int construct_devpath_if_missing(Config* cfg) -{ - // Windows doesn't allow us to request specific disk paths when mapping an - // image. This will just be used by rbd-wnbd and wnbd as an identifier. - if (cfg->devpath.empty()) { - if (cfg->imgname.empty()) { - derr << "Missing image name." << dendl; - return -EINVAL; - } - - if (!cfg->poolname.empty()) { - cfg->devpath += cfg->poolname; - cfg->devpath += '/'; - } - if (!cfg->nsname.empty()) { - cfg->devpath += cfg->nsname; - cfg->devpath += '/'; - } - - cfg->devpath += cfg->imgname; - - if (!cfg->snapname.empty()) { - cfg->devpath += '@'; - cfg->devpath += cfg->snapname; - } - } - - return 0; -} - boost::intrusive_ptr do_global_init( int argc, const char *argv[], Config *cfg) { @@ -1233,16 +1107,6 @@ static int wait_mapped_disk(Config *cfg) static int do_map(Config *cfg) { - int r; - - librados::Rados rados; - librbd::RBD rbd; - librados::IoCtx io_ctx; - librbd::Image image; - librbd::image_info_t info; - HANDLE parent_pipe_handle = INVALID_HANDLE_VALUE; - int err = 0; - if (g_conf()->daemonize && cfg->parent_pipe.empty()) { r = send_map_request(get_cli_args()); if (r < 0) { @@ -1254,85 +1118,18 @@ static int do_map(Config *cfg) dout(0) << "Mapping RBD image: " << cfg->devpath << dendl; - r = rados.init_with_context(g_ceph_context); + librados::Rados rados; + int r = rados.init_with_context(g_ceph_context); if (r < 0) { derr << "rbd-wnbd: couldn't initialize rados: " << cpp_strerror(r) << dendl; - goto close_ret; - } - - r = rados.connect(); - if (r < 0) { - derr << "rbd-wnbd: couldn't connect to rados: " << cpp_strerror(r) - << dendl; - goto close_ret; - } - - r = rados.ioctx_create(cfg->poolname.c_str(), io_ctx); - if (r < 0) { - derr << "rbd-wnbd: couldn't create IO context: " << cpp_strerror(r) - << dendl; - goto close_ret; - } - - io_ctx.set_namespace(cfg->nsname); - - r = rbd.open(io_ctx, image, cfg->imgname.c_str()); - if (r < 0) { - derr << "rbd-wnbd: couldn't open rbd image: " << cpp_strerror(r) - << dendl; - goto close_ret; - } - - if (cfg->exclusive) { - r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE); - if (r < 0) { - derr << "rbd-wnbd: failed to acquire exclusive lock: " << cpp_strerror(r) - << dendl; - goto close_ret; - } - } - - if (!cfg->snapname.empty()) { - r = image.snap_set(cfg->snapname.c_str()); - if (r < 0) { - derr << "rbd-wnbd: couldn't use snapshot: " << cpp_strerror(r) - << dendl; - goto close_ret; - } - } - - r = image.stat(info, sizeof(info)); - if (r < 0) - goto close_ret; - - if (info.size > _UI64_MAX) { - r = -EFBIG; - derr << "rbd-wnbd: image is too large (" << byte_u_t(info.size) - << ", max is " << byte_u_t(_UI64_MAX) << ")" << dendl; - goto close_ret; + return r; } - // We're storing mapping details in the registry even for non-persistent - // mappings. This allows us to easily retrieve mapping details such - // as the rbd pool or admin socket path. - // We're cleaning up the registry entry when the non-persistent mapping - // gets disconnected or when the ceph service restarts. - r = save_config_to_registry(cfg); - if (r < 0) - goto close_ret; - - handler = new WnbdHandler(image, cfg->devpath, - info.size / RBD_WNBD_BLKSIZE, - RBD_WNBD_BLKSIZE, - !cfg->snapname.empty() || cfg->readonly, - g_conf().get_val("rbd_cache"), - cfg->io_req_workers, - cfg->io_reply_workers); - r = handler->start(); + RbdMapping rbd_mapping(*cfg, rados, get_cli_args()); + r = rbd_mapping.start(); if (r) { - r = r == ERROR_ALREADY_EXISTS ? -EEXIST : -EINVAL; - goto close_ret; + return r; } // TODO: consider substracting the time it took to perform the @@ -1342,75 +1139,8 @@ static int do_map(Config *cfg) goto close_ret; } - // We're informing the parent processes that the initialization - // was successful. - if (!cfg->parent_pipe.empty()) { - parent_pipe_handle = CreateFile( - cfg->parent_pipe.c_str(), GENERIC_WRITE, 0, NULL, - OPEN_EXISTING, 0, NULL); - if (parent_pipe_handle == INVALID_HANDLE_VALUE) { - derr << "Could not open parent pipe: " << win32_strerror(err) << dendl; - } else if (!WriteFile(parent_pipe_handle, "a", 1, NULL, NULL)) { - // TODO: consider exiting in this case. The parent didn't wait for us, - // maybe it was killed after a timeout. - err = GetLastError(); - derr << "Failed to communicate with the parent: " - << win32_strerror(err) << dendl; - } else { - dout(5) << __func__ << ": submitted parent notification." << dendl; - } - - if (parent_pipe_handle != INVALID_HANDLE_VALUE) - CloseHandle(parent_pipe_handle); - - global_init_postfork_finish(g_ceph_context); - } - - { - uint64_t watch_handle; - WNBDWatchCtx watch_ctx(io_ctx, handler, image, info.size); - r = image.update_watch(&watch_ctx, &watch_handle); - if (r < 0) { - derr << __func__ << ": update_watch failed with error: " - << cpp_strerror(r) << dendl; - - handler->shutdown(); - goto close_ret; - } - - handler->wait(); - - r = image.update_unwatch(watch_handle); - if (r < 0) - derr << __func__ << ": update_unwatch failed with error: " - << cpp_strerror(r) << dendl; - - handler->shutdown(); - } - -close_ret: - // The registry record shouldn't be removed for (already) running mappings. - if (!cfg->persistent) { - dout(5) << __func__ << ": cleaning up non-persistent mapping: " - << cfg->devpath << dendl; - r = remove_config_from_registry(cfg); - if (r) { - derr << __func__ << ": could not clean up non-persistent mapping: " - << cfg->devpath << dendl; - } - } - - std::unique_lock l{shutdown_lock}; - - image.close(); - io_ctx.close(); - rados.shutdown(); - if (handler) { - delete handler; - handler = nullptr; - } - - return r; + dout(0) << "Successfully mapped RBD image: " << cfg->devpath << dendl; + return rbd_mapping.wait(); } static int do_unmap(Config *cfg, bool unregister) diff --git a/src/tools/rbd_wnbd/rbd_wnbd.h b/src/tools/rbd_wnbd/rbd_wnbd.h index ac24e9de4aaab..ba6280031a364 100644 --- a/src/tools/rbd_wnbd/rbd_wnbd.h +++ b/src/tools/rbd_wnbd/rbd_wnbd.h @@ -20,19 +20,13 @@ #include "include/compat.h" #include "common/win32/registry.h" -#include "wnbd_handler.h" +#include "rbd_mapping_config.h" +#include "rbd_mapping.h" -#define SERVICE_REG_KEY "SYSTEM\\CurrentControlSet\\Services\\rbd-wnbd" #define SERVICE_PIPE_NAME "\\\\.\\pipe\\rbd-wnbd" #define SERVICE_PIPE_TIMEOUT_MS 5000 #define SERVICE_PIPE_BUFFSZ 4096 -#define DEFAULT_MAP_TIMEOUT_MS 30000 - -#define RBD_WNBD_BLKSIZE 512UL - -#define DEFAULT_SERVICE_START_TIMEOUT 120 -#define DEFAULT_IMAGE_MAP_TIMEOUT 20 #define DISK_STATUS_POLLING_INTERVAL_MS 500 #define HELP_INFO 1 @@ -41,52 +35,6 @@ #define WNBD_STATUS_ACTIVE "active" #define WNBD_STATUS_INACTIVE "inactive" -#define DEFAULT_SERVICE_THREAD_COUNT 8 - -struct Config { - bool exclusive = false; - bool readonly = false; - - std::string parent_pipe; - - std::string poolname; - std::string nsname; - std::string imgname; - std::string snapname; - std::string devpath; - - std::string format; - bool pretty_format = false; - - bool hard_disconnect = false; - int soft_disconnect_timeout = DEFAULT_SOFT_REMOVE_TIMEOUT; - bool hard_disconnect_fallback = true; - - int service_start_timeout = DEFAULT_SERVICE_START_TIMEOUT; - int image_map_timeout = DEFAULT_IMAGE_MAP_TIMEOUT; - bool remap_failure_fatal = false; - bool adapter_monitoring_enabled = false; - - // TODO: consider moving those fields to a separate structure. Those - // provide connection information without actually being configurable. - // The disk number is provided by Windows. - int disk_number = -1; - int pid = 0; - std::string serial_number; - bool active = false; - bool wnbd_mapped = false; - std::string command_line; - std::string admin_sock_path; - - WnbdLogLevel wnbd_log_level = WnbdLogLevelInfo; - int io_req_workers = DEFAULT_IO_WORKER_COUNT; - int io_reply_workers = DEFAULT_IO_WORKER_COUNT; - int service_thread_count = DEFAULT_SERVICE_THREAD_COUNT; - - // register the mapping, recreating it when the Ceph service starts. - bool persistent = true; -}; - enum Command { None, Connect, @@ -118,11 +66,6 @@ int restart_registered_mappings( int worker_count, int total_timeout, int image_map_timeout); int map_device_using_suprocess(std::string command_line); -int construct_devpath_if_missing(Config* cfg); -int save_config_to_registry(Config* cfg); -int remove_config_from_registry(Config* cfg); -int load_mapping_config_from_registry(std::string devpath, Config* cfg); - BOOL WINAPI console_handler_routine(DWORD dwCtrlType); static int parse_args(std::vector& args, diff --git a/src/tools/rbd_wnbd/wnbd_handler.h b/src/tools/rbd_wnbd/wnbd_handler.h index c1ab5676bb85b..f3b3e4341b01a 100644 --- a/src/tools/rbd_wnbd/wnbd_handler.h +++ b/src/tools/rbd_wnbd/wnbd_handler.h @@ -27,8 +27,6 @@ // TODO: make this configurable. #define RBD_WNBD_MAX_TRANSFER 2 * 1024 * 1024 #define SOFT_REMOVE_RETRY_INTERVAL 2 -#define DEFAULT_SOFT_REMOVE_TIMEOUT 15 -#define DEFAULT_IO_WORKER_COUNT 4 // Not defined by mingw. #ifndef SCSI_ADSENSE_UNRECOVERED_ERROR -- 2.39.5