From 54853a5ae760c639aee64852b9785626622795ab Mon Sep 17 00:00:00 2001 From: Dongsheng Yang Date: Wed, 15 Feb 2023 11:16:02 +0000 Subject: [PATCH] tools/rbd: add support of new device type for ubbd ubbd (Userspace Backend Block Device) is a method to build block device and handle IO by process started in userspace. That means we can provide a generic block device to user and handle IO requests by librbd in linux. This way, we can allow user to use rbd image as a linux block device supporting full image features, especial the journaling feature, which is not supported by krbd. For more information: https://github.com/DataTravelGuide/ubbd Signed-off-by: Dongsheng Yang --- CMakeLists.txt | 8 ++ cmake/modules/BuildUBBD.cmake | 22 +++ doc/man/8/rbd.rst | 2 +- src/include/config-h.in.cmake | 3 + src/test/cli/rbd/help.t | 10 +- src/tools/rbd/CMakeLists.txt | 14 +- src/tools/rbd/action/Device.cc | 16 ++- src/tools/rbd/action/Ubbd.cc | 252 +++++++++++++++++++++++++++++++++ 8 files changed, 319 insertions(+), 8 deletions(-) create mode 100644 cmake/modules/BuildUBBD.cmake create mode 100644 src/tools/rbd/action/Ubbd.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 0724fcf5075f..729bae5135ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -270,6 +270,14 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF "WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF) +CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF + "WITH_RBD" OFF) + +if(WITH_RBD_UBBD) + include(BuildUBBD) + build_ubbd() +endif() + if(WITH_BLUESTORE_PMEM) set(HAVE_BLUESTORE_PMEM ON) endif() diff --git a/cmake/modules/BuildUBBD.cmake b/cmake/modules/BuildUBBD.cmake new file mode 100644 index 000000000000..6e11dc42257b --- /dev/null +++ b/cmake/modules/BuildUBBD.cmake @@ -0,0 +1,22 @@ +function(build_ubbd) + include(FindMake) + find_make("MAKE_EXECUTABLE" "make_cmd") + + include(ExternalProject) + ExternalProject_Add(ubbd_ext + UPDATE_COMMAND "" # this disables rebuild on each run + GIT_REPOSITORY "https://github.com/DataTravelGuide/ubbd.git" + GIT_CONFIG advice.detachedHead=false + GIT_SHALLOW 1 + GIT_TAG "v0.1.4" + SOURCE_DIR "${CMAKE_BINARY_DIR}/src/ubbd" + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND ${make_cmd} libubbd + INSTALL_COMMAND "" + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + LOG_MERGED_STDOUTERR ON + LOG_OUTPUT_ON_FAILURE ON) +endfunction() diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 59e346209753..c60052ed3c7c 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -260,7 +260,7 @@ Commands :command:`device map` [-t | --device-type *device-type*] [--cookie *device-cookie*] [--show-cookie] [--snap-id *snap-id*] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec* Map the specified image to a block device via the rbd kernel module - (default) or other supported device (*nbd* on Linux or *ggate* on + (default) or other supported device (*nbd* or *ubbd* on Linux or *ggate* on FreeBSD). The --options argument is a comma separated list of device type diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index cc9ad0ec7f7b..a9bbb9dfd2ec 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -148,6 +148,9 @@ /* define if kernel rbd enabled */ #cmakedefine WITH_KRBD +/* define if rbd ubbd enabled */ +#cmakedefine WITH_RBD_UBBD + /* define if key-value-store is enabled */ #cmakedefine WITH_KVS diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 8d8d30273810..a80c0fb491ed 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -594,7 +594,7 @@ ]) Optional arguments - -t [ --device-type ] arg device type [ggate, krbd (default), nbd] + -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd] -p [ --pool ] arg pool name --namespace arg namespace name --image arg image name @@ -625,7 +625,7 @@ p-name>] or Optional arguments - -t [ --device-type ] arg device type [ggate, krbd (default), nbd] + -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd] -p [ --pool ] arg pool name --namespace arg namespace name --image arg image name @@ -640,7 +640,7 @@ List mapped rbd images. Optional arguments - -t [ --device-type ] arg device type [ggate, krbd (default), nbd] + -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd] --format arg output format (plain, json, or xml) [default: plain] --pretty-format pretty formatting (json and xml) @@ -662,7 +662,7 @@ ]) Optional arguments - -t [ --device-type ] arg device type [ggate, krbd (default), nbd] + -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd] -p [ --pool ] arg pool name --namespace arg namespace name --image arg image name @@ -691,7 +691,7 @@ p-name>] or Optional arguments - -t [ --device-type ] arg device type [ggate, krbd (default), nbd] + -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd] -p [ --pool ] arg pool name --namespace arg namespace name --image arg image name diff --git a/src/tools/rbd/CMakeLists.txt b/src/tools/rbd/CMakeLists.txt index 5a895354d536..19b4e806a752 100644 --- a/src/tools/rbd/CMakeLists.txt +++ b/src/tools/rbd/CMakeLists.txt @@ -52,6 +52,7 @@ set(rbd_srcs action/Status.cc action/TrashPurgeSchedule.cc action/Trash.cc + action/Ubbd.cc action/Watch.cc action/Wnbd.cc) @@ -73,8 +74,19 @@ if(CURSES_FOUND) target_link_libraries(rbd ${CURSES_LIBRARIES}) endif() if(WITH_KRBD) - target_link_libraries(rbd + target_link_libraries(rbd krbd) endif() +if(WITH_RBD_UBBD) + target_link_directories(rbd + PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/lib/) + target_include_directories(rbd + PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/include) + target_include_directories(rbd + PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/include/ubbd-headers/) + target_link_libraries(rbd + ubbd) + add_dependencies(rbd ubbd_ext) +endif() install(TARGETS rbd DESTINATION bin) diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc index 878081438377..d306e2dacf8e 100644 --- a/src/tools/rbd/action/Device.cc +++ b/src/tools/rbd/action/Device.cc @@ -33,6 +33,7 @@ DECLARE_DEVICE_OPERATIONS(ggate); DECLARE_DEVICE_OPERATIONS(kernel); DECLARE_DEVICE_OPERATIONS(nbd); DECLARE_DEVICE_OPERATIONS(wnbd); +DECLARE_DEVICE_OPERATIONS(ubbd); namespace device { @@ -83,11 +84,20 @@ const DeviceOperations wnbd_operations = { wnbd::execute_detach, }; +const DeviceOperations ubbd_operations = { + ubbd::execute_list, + ubbd::execute_map, + ubbd::execute_unmap, + ubbd::execute_attach, + ubbd::execute_detach, +}; + enum device_type_t { DEVICE_TYPE_GGATE, DEVICE_TYPE_KRBD, DEVICE_TYPE_NBD, DEVICE_TYPE_WNBD, + DEVICE_TYPE_UBBD, }; struct DeviceType {}; @@ -107,6 +117,8 @@ void validate(boost::any& v, const std::vector& values, v = boost::any(DEVICE_TYPE_GGATE); } else if (s == "krbd") { v = boost::any(DEVICE_TYPE_KRBD); + } else if (s == "ubbd") { + v = boost::any(DEVICE_TYPE_UBBD); #endif /* _WIN32 */ } else { throw po::validation_error(po::validation_error::invalid_option_value); @@ -119,7 +131,7 @@ void add_device_type_option(po::options_description *options) { #ifdef _WIN32 "device type [wnbd]"); #else - "device type [ggate, krbd (default), nbd]"); + "device type [ggate, krbd (default), nbd, ubbd]"); #endif } @@ -150,6 +162,8 @@ const DeviceOperations *get_device_operations(const po::variables_map &vm) { return &nbd_operations; case DEVICE_TYPE_WNBD: return &wnbd_operations; + case DEVICE_TYPE_UBBD: + return &ubbd_operations; default: ceph_abort(); return nullptr; diff --git a/src/tools/rbd/action/Ubbd.cc b/src/tools/rbd/action/Ubbd.cc new file mode 100644 index 000000000000..621f1ba38038 --- /dev/null +++ b/src/tools/rbd/action/Ubbd.cc @@ -0,0 +1,252 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/errno.h" +#include "common/Formatter.h" +#include "common/TextTable.h" +#include "tools/rbd/Shell.h" +#include "tools/rbd/Utils.h" + +#include +#include +#include +#include + +#if defined(WITH_RBD_UBBD) +#include +#endif + +namespace rbd { +namespace action { +namespace ubbd { + +namespace at = argument_types; +namespace po = boost::program_options; + +int execute_list(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { +#if defined(WITH_RBD_UBBD) + ubbdd_mgmt_rsp list_rsp; + ubbd_list_options list_opts = { .type = UBBD_DEV_TYPE_RBD }; + + int r = ubbd_list(&list_opts, &list_rsp); + if (r < 0) { + std::cerr << "rbd: ubbd_list failed: " << cpp_strerror(r) << std::endl; + return r; + } + + at::Format::Formatter f; + r = utils::get_formatter(vm, &f); + if (r < 0) { + return r; + } + + TextTable tbl; + + if (f) { + f->open_array_section("devices"); + } else { + tbl.define_column("id", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("namespace", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("image", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("device", TextTable::LEFT, TextTable::LEFT); + } + + for (int i = 0; i < list_rsp.list.dev_num; i++) { + ubbd_info_options info_opts = { .ubbdid = list_rsp.list.dev_list[i] }; + ubbdd_mgmt_rsp info_rsp; + ubbdd_mgmt_rsp_dev_info *mgmt_dev_info = &info_rsp.dev_info; + ubbd_dev_info *dev_info = &mgmt_dev_info->dev_info; + + r = ubbd_device_info(&info_opts, &info_rsp); + if (r < 0) { + std::cerr << "ubbd_device_info for /dev/ubbd" << info_opts.ubbdid + << " failed: " << cpp_strerror(r) << std::endl; + return r; + } + + if (f) { + f->open_object_section("device"); + f->dump_int("id", mgmt_dev_info->devid); + f->dump_string("pool", dev_info->generic_dev.info.rbd.pool); + f->dump_string("namespace", dev_info->generic_dev.info.rbd.ns); + f->dump_string("image", dev_info->generic_dev.info.rbd.image); + f->dump_string("snap", dev_info->generic_dev.info.rbd.snap); + f->dump_string("device", "/dev/ubbd" + std::to_string(mgmt_dev_info->devid)); + f->close_section(); + } else { + tbl << mgmt_dev_info->devid << dev_info->generic_dev.info.rbd.pool + << dev_info->generic_dev.info.rbd.ns << dev_info->generic_dev.info.rbd.image + << dev_info->generic_dev.info.rbd.snap + << "/dev/ubbd" + std::to_string(mgmt_dev_info->devid) + << TextTable::endrow; + } + } + + if (f) { + f->close_section(); // devices + f->flush(std::cout); + } else { + std::cout << tbl; + } + + return 0; +#else + std::cerr << "rbd: ubbd device is not supported" << std::endl; + return -EOPNOTSUPP; +#endif +} + +int execute_map(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { +#if defined(WITH_RBD_UBBD) + size_t arg_index = 0; + std::string pool_name; + std::string nspace_name; + std::string image_name; + std::string snap_name; + ubbdd_mgmt_rsp rsp; + ubbd_map_options opts = { 0 }; + + int r = utils::get_pool_image_snapshot_names( + vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name, + &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED, + utils::SPEC_VALIDATION_NONE); + if (r < 0) { + return r; + } + opts.type = opts.generic_dev.opts.type = "rbd"; + opts.generic_dev.opts.rbd.pool = pool_name.c_str(); + opts.generic_dev.opts.rbd.ns = nspace_name.c_str(); + opts.generic_dev.opts.rbd.image = image_name.c_str(); + opts.generic_dev.opts.rbd.snap = snap_name.c_str(); + + if (vm["read-only"].as()) { + opts.read_only = true; + } + if (vm["exclusive"].as()) { + opts.generic_dev.opts.rbd.exclusive = true; + } + if (vm["quiesce"].as()) { + opts.generic_dev.opts.rbd.quiesce = true; + } + if (vm.count("quiesce-hook")) { + opts.generic_dev.opts.rbd.quiesce_hook = + vm["quiesce-hook"].as().c_str(); + } + + r = ubbd_map(&opts, &rsp); + if (r < 0) { + std::cerr << "rbd: ubbd_map failed: " << cpp_strerror(r) << std::endl; + return r; + } + + std::cout << rsp.add.path << std::endl; + + return 0; +#else + std::cerr << "rbd: ubbd device is not supported" << std::endl; + return -EOPNOTSUPP; +#endif +} + +#if defined(WITH_RBD_UBBD) +static int parse_unmap_options(const std::string &options_string, + ubbd_unmap_options *unmap_opts) +{ + char *options = strdup(options_string.c_str()); + BOOST_SCOPE_EXIT(options) { + free(options); + } BOOST_SCOPE_EXIT_END; + + for (char *this_char = strtok(options, ", "); + this_char != NULL; + this_char = strtok(NULL, ",")) { + char *value_char; + + if ((value_char = strchr(this_char, '=')) != NULL) + *value_char++ = '\0'; + + if (!strcmp(this_char, "force")) { + unmap_opts->force = true; + } else { + std::cerr << "rbd: unknown ubbd unmap option '" << this_char << "'" + << std::endl; + return -EINVAL; + } + } + + return 0; +} +#endif + +int execute_unmap(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { +#if defined(WITH_RBD_UBBD) + std::string device_name = utils::get_positional_argument(vm, 0); + ubbdd_mgmt_rsp rsp; + ubbd_unmap_options opts = { 0 }; + struct stat sb; + int r; + + if (!boost::starts_with(device_name, "/dev/ubbd")) { + std::cerr << "rbd: ubbd unmap requires device path (/dev/ubbdX)" << std::endl; + return -EINVAL; + } + + if (stat(device_name.c_str(), &sb) < 0 || !S_ISBLK(sb.st_mode)) { + std::cerr << "rbd: '" << device_name << "' is not a block device" << std::endl; + return -EINVAL; + } + + device_name.erase(0, 9); + opts.ubbdid = stoi(device_name); + + if (vm.count("options")) { + for (auto &options : vm["options"].as>()) { + r = parse_unmap_options(options, &opts); + if (r < 0) { + std::cerr << "rbd: couldn't parse ubbd unmap options" << std::endl; + return r; + } + } + } + + r = ubbd_unmap(&opts, &rsp); + if (r < 0) { + std::cerr << "rbd: ubbd_unmap failed: " << cpp_strerror(r) << std::endl; + return r; + } + + return 0; +#else + std::cerr << "rbd: ubbd device is not supported" << std::endl; + return -EOPNOTSUPP; +#endif +} + +int execute_attach(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { +#if defined(WITH_RBD_UBBD) + std::cerr << "rbd: ubbd does not support attach" << std::endl; +#else + std::cerr << "rbd: ubbd device is not supported" << std::endl; +#endif + return -EOPNOTSUPP; +} + +int execute_detach(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { +#if defined(WITH_RBD_UBBD) + std::cerr << "rbd: ubbd does not support detach" << std::endl; +#else + std::cerr << "rbd: ubbd device is not supported" << std::endl; +#endif + return -EOPNOTSUPP; +} + +} // namespace ubbd +} // namespace action +} // namespace rbd -- 2.47.3