]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
tools/rbd: add support of new device type for ubbd 50341/head
authorDongsheng Yang <dongsheng.yang.linux@gmail.com>
Wed, 15 Feb 2023 11:16:02 +0000 (11:16 +0000)
committerDongsheng Yang <dongsheng.yang.linux@gmail.com>
Thu, 18 May 2023 12:35:39 +0000 (12:35 +0000)
ubbd (Userspace Backend Block Device) is a method to build block device
and handle IO by process started in userspace. That means we can provide
a generic block device to user and handle IO requests by librbd
in linux.

This way, we can allow user to use rbd image as a linux block device
supporting full image features, especial the journaling feature, which
is not supported by krbd.

For more information: https://github.com/DataTravelGuide/ubbd

Signed-off-by: Dongsheng Yang <dongsheng.yang.linux@gmail.com>
CMakeLists.txt
cmake/modules/BuildUBBD.cmake [new file with mode: 0644]
doc/man/8/rbd.rst
src/include/config-h.in.cmake
src/test/cli/rbd/help.t
src/tools/rbd/CMakeLists.txt
src/tools/rbd/action/Device.cc
src/tools/rbd/action/Ubbd.cc [new file with mode: 0644]

index 0724fcf5075f643de75651d7ee8a6be64a642021..729bae5135ae022dd026e47615006aa2be685a4a 100644 (file)
@@ -270,6 +270,14 @@ CMAKE_DEPENDENT_OPTION(WITH_RBD_SSD_CACHE "Enable librbd persistent write back c
 CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
   "WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF)
 
+CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF
+    "WITH_RBD" OFF)
+
+if(WITH_RBD_UBBD)
+  include(BuildUBBD)
+  build_ubbd()
+endif()
+
 if(WITH_BLUESTORE_PMEM)
   set(HAVE_BLUESTORE_PMEM ON)
 endif()
diff --git a/cmake/modules/BuildUBBD.cmake b/cmake/modules/BuildUBBD.cmake
new file mode 100644 (file)
index 0000000..6e11dc4
--- /dev/null
@@ -0,0 +1,22 @@
+function(build_ubbd)
+  include(FindMake)
+  find_make("MAKE_EXECUTABLE" "make_cmd")
+
+  include(ExternalProject)
+  ExternalProject_Add(ubbd_ext
+    UPDATE_COMMAND "" # this disables rebuild on each run
+    GIT_REPOSITORY "https://github.com/DataTravelGuide/ubbd.git"
+    GIT_CONFIG advice.detachedHead=false
+    GIT_SHALLOW 1
+    GIT_TAG "v0.1.4"
+    SOURCE_DIR "${CMAKE_BINARY_DIR}/src/ubbd"
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ${make_cmd} libubbd
+    INSTALL_COMMAND ""
+    LOG_CONFIGURE ON
+    LOG_BUILD ON
+    LOG_INSTALL ON
+    LOG_MERGED_STDOUTERR ON
+    LOG_OUTPUT_ON_FAILURE ON)
+endfunction()
index 59e3462097530ea38c7fc967bd60b28e81ceb112..c60052ed3c7cbd669c73eefee18807964b1907a7 100644 (file)
@@ -260,7 +260,7 @@ Commands
 
 :command:`device map` [-t | --device-type *device-type*] [--cookie *device-cookie*] [--show-cookie] [--snap-id *snap-id*] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec*
   Map the specified image to a block device via the rbd kernel module
-  (default) or other supported device (*nbd* on Linux or *ggate* on
+  (default) or other supported device (*nbd* or *ubbd* on Linux or *ggate* on
   FreeBSD).
 
   The --options argument is a comma separated list of device type
index cc9ad0ec7f7bd8ba4c862a9286812e13c28b2dff..a9bbb9dfd2ec2a6839e68db6d2f23d5231d59121 100644 (file)
 /* define if kernel rbd enabled */
 #cmakedefine WITH_KRBD
 
+/* define if rbd ubbd enabled */
+#cmakedefine WITH_RBD_UBBD
+
 /* define if key-value-store is enabled */
 #cmakedefine WITH_KVS
 
index 8d8d30273810179dd3e2b89666096d2ddf52531f..a80c0fb491edbf0c41034ec3b33eaf8d6bfe2593 100644 (file)
                              ])
   
   Optional arguments
-    -t [ --device-type ] arg device type [ggate, krbd (default), nbd]
+    -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd]
     -p [ --pool ] arg        pool name
     --namespace arg          namespace name
     --image arg              image name
                                     p-name>] or <device-path>
   
   Optional arguments
-    -t [ --device-type ] arg        device type [ggate, krbd (default), nbd]
+    -t [ --device-type ] arg        device type [ggate, krbd (default), nbd, ubbd]
     -p [ --pool ] arg               pool name
     --namespace arg                 namespace name
     --image arg                     image name
   List mapped rbd images.
   
   Optional arguments
-    -t [ --device-type ] arg device type [ggate, krbd (default), nbd]
+    -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd]
     --format arg             output format (plain, json, or xml) [default: plain]
     --pretty-format          pretty formatting (json and xml)
   
                              ])
   
   Optional arguments
-    -t [ --device-type ] arg device type [ggate, krbd (default), nbd]
+    -t [ --device-type ] arg device type [ggate, krbd (default), nbd, ubbd]
     -p [ --pool ] arg        pool name
     --namespace arg          namespace name
     --image arg              image name
                                     p-name>] or <device-path>
   
   Optional arguments
-    -t [ --device-type ] arg        device type [ggate, krbd (default), nbd]
+    -t [ --device-type ] arg        device type [ggate, krbd (default), nbd, ubbd]
     -p [ --pool ] arg               pool name
     --namespace arg                 namespace name
     --image arg                     image name
index 5a895354d536519e9890061154a785b0f62b68f6..19b4e806a752ff038b44982c8619fe017f97a56f 100644 (file)
@@ -52,6 +52,7 @@ set(rbd_srcs
   action/Status.cc
   action/TrashPurgeSchedule.cc
   action/Trash.cc
+  action/Ubbd.cc
   action/Watch.cc
   action/Wnbd.cc)
 
@@ -73,8 +74,19 @@ if(CURSES_FOUND)
   target_link_libraries(rbd ${CURSES_LIBRARIES})
 endif()
 if(WITH_KRBD)
-  target_link_libraries(rbd 
+  target_link_libraries(rbd
     krbd)
 endif()
+if(WITH_RBD_UBBD)
+  target_link_directories(rbd
+    PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/lib/)
+  target_include_directories(rbd
+    PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/include)
+  target_include_directories(rbd
+    PRIVATE ${CMAKE_BINARY_DIR}/src/ubbd/include/ubbd-headers/)
+  target_link_libraries(rbd
+    ubbd)
+  add_dependencies(rbd ubbd_ext)
+endif()
 
 install(TARGETS rbd DESTINATION bin)
index 878081438377c651dee24442b17ab7ac4d433eb3..d306e2dacf8ec18407d114bf7df31dfe486202ad 100644 (file)
@@ -33,6 +33,7 @@ DECLARE_DEVICE_OPERATIONS(ggate);
 DECLARE_DEVICE_OPERATIONS(kernel);
 DECLARE_DEVICE_OPERATIONS(nbd);
 DECLARE_DEVICE_OPERATIONS(wnbd);
+DECLARE_DEVICE_OPERATIONS(ubbd);
 
 namespace device {
 
@@ -83,11 +84,20 @@ const DeviceOperations wnbd_operations = {
   wnbd::execute_detach,
 };
 
+const DeviceOperations ubbd_operations = {
+  ubbd::execute_list,
+  ubbd::execute_map,
+  ubbd::execute_unmap,
+  ubbd::execute_attach,
+  ubbd::execute_detach,
+};
+
 enum device_type_t {
   DEVICE_TYPE_GGATE,
   DEVICE_TYPE_KRBD,
   DEVICE_TYPE_NBD,
   DEVICE_TYPE_WNBD,
+  DEVICE_TYPE_UBBD,
 };
 
 struct DeviceType {};
@@ -107,6 +117,8 @@ void validate(boost::any& v, const std::vector<std::string>& values,
     v = boost::any(DEVICE_TYPE_GGATE);
   } else if (s == "krbd") {
     v = boost::any(DEVICE_TYPE_KRBD);
+  } else if (s == "ubbd") {
+    v = boost::any(DEVICE_TYPE_UBBD);
   #endif /* _WIN32 */
   } else {
     throw po::validation_error(po::validation_error::invalid_option_value);
@@ -119,7 +131,7 @@ void add_device_type_option(po::options_description *options) {
 #ifdef _WIN32
      "device type [wnbd]");
 #else
-     "device type [ggate, krbd (default), nbd]");
+     "device type [ggate, krbd (default), nbd, ubbd]");
 #endif
 }
 
@@ -150,6 +162,8 @@ const DeviceOperations *get_device_operations(const po::variables_map &vm) {
     return &nbd_operations;
   case DEVICE_TYPE_WNBD:
     return &wnbd_operations;
+  case DEVICE_TYPE_UBBD:
+    return &ubbd_operations;
   default:
     ceph_abort();
     return nullptr;
diff --git a/src/tools/rbd/action/Ubbd.cc b/src/tools/rbd/action/Ubbd.cc
new file mode 100644 (file)
index 0000000..621f1ba
--- /dev/null
@@ -0,0 +1,252 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/scope_exit.hpp>
+#include <iostream>
+#include <sys/stat.h>
+
+#if defined(WITH_RBD_UBBD)
+#include <libubbd.h>
+#endif
+
+namespace rbd {
+namespace action {
+namespace ubbd {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+#if defined(WITH_RBD_UBBD)
+  ubbdd_mgmt_rsp list_rsp;
+  ubbd_list_options list_opts = { .type = UBBD_DEV_TYPE_RBD };
+
+  int r = ubbd_list(&list_opts, &list_rsp);
+  if (r < 0) {
+    std::cerr << "rbd: ubbd_list failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  at::Format::Formatter f;
+  r = utils::get_formatter(vm, &f);
+  if (r < 0) {
+    return r;
+  }
+
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("devices");
+  } else {
+    tbl.define_column("id", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("namespace", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("image", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("device", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (int i = 0; i < list_rsp.list.dev_num; i++) {
+    ubbd_info_options info_opts = { .ubbdid = list_rsp.list.dev_list[i] };
+    ubbdd_mgmt_rsp info_rsp;
+    ubbdd_mgmt_rsp_dev_info *mgmt_dev_info = &info_rsp.dev_info;
+    ubbd_dev_info *dev_info = &mgmt_dev_info->dev_info;
+
+    r = ubbd_device_info(&info_opts, &info_rsp);
+    if (r < 0) {
+      std::cerr << "ubbd_device_info for /dev/ubbd" << info_opts.ubbdid
+                << " failed: " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+
+    if (f) {
+      f->open_object_section("device");
+      f->dump_int("id", mgmt_dev_info->devid);
+      f->dump_string("pool", dev_info->generic_dev.info.rbd.pool);
+      f->dump_string("namespace", dev_info->generic_dev.info.rbd.ns);
+      f->dump_string("image", dev_info->generic_dev.info.rbd.image);
+      f->dump_string("snap", dev_info->generic_dev.info.rbd.snap);
+      f->dump_string("device", "/dev/ubbd" + std::to_string(mgmt_dev_info->devid));
+      f->close_section();
+    } else {
+      tbl << mgmt_dev_info->devid << dev_info->generic_dev.info.rbd.pool
+          << dev_info->generic_dev.info.rbd.ns << dev_info->generic_dev.info.rbd.image
+          << dev_info->generic_dev.info.rbd.snap
+          << "/dev/ubbd" + std::to_string(mgmt_dev_info->devid)
+          << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section(); // devices
+    f->flush(std::cout);
+  } else {
+    std::cout << tbl;
+  }
+
+  return 0;
+#else
+  std::cerr << "rbd: ubbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+int execute_map(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+#if defined(WITH_RBD_UBBD)
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string nspace_name;
+  std::string image_name;
+  std::string snap_name;
+  ubbdd_mgmt_rsp rsp;
+  ubbd_map_options opts = { 0 };
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+  opts.type = opts.generic_dev.opts.type = "rbd";
+  opts.generic_dev.opts.rbd.pool = pool_name.c_str();
+  opts.generic_dev.opts.rbd.ns = nspace_name.c_str();
+  opts.generic_dev.opts.rbd.image = image_name.c_str();
+  opts.generic_dev.opts.rbd.snap = snap_name.c_str();
+
+  if (vm["read-only"].as<bool>()) {
+    opts.read_only = true;
+  }
+  if (vm["exclusive"].as<bool>()) {
+    opts.generic_dev.opts.rbd.exclusive = true;
+  }
+  if (vm["quiesce"].as<bool>()) {
+    opts.generic_dev.opts.rbd.quiesce = true;
+  }
+  if (vm.count("quiesce-hook")) {
+    opts.generic_dev.opts.rbd.quiesce_hook =
+      vm["quiesce-hook"].as<std::string>().c_str();
+  }
+
+  r = ubbd_map(&opts, &rsp);
+  if (r < 0) {
+    std::cerr << "rbd: ubbd_map failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  std::cout << rsp.add.path << std::endl;
+
+  return 0;
+#else
+  std::cerr << "rbd: ubbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+#if defined(WITH_RBD_UBBD)
+static int parse_unmap_options(const std::string &options_string,
+                               ubbd_unmap_options *unmap_opts)
+{
+  char *options = strdup(options_string.c_str());
+  BOOST_SCOPE_EXIT(options) {
+    free(options);
+  } BOOST_SCOPE_EXIT_END;
+
+  for (char *this_char = strtok(options, ", ");
+       this_char != NULL;
+       this_char = strtok(NULL, ",")) {
+    char *value_char;
+
+    if ((value_char = strchr(this_char, '=')) != NULL)
+      *value_char++ = '\0';
+
+    if (!strcmp(this_char, "force")) {
+      unmap_opts->force = true;
+    } else {
+      std::cerr << "rbd: unknown ubbd unmap option '" << this_char << "'"
+                << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  return 0;
+}
+#endif
+
+int execute_unmap(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+#if defined(WITH_RBD_UBBD)
+  std::string device_name = utils::get_positional_argument(vm, 0);
+  ubbdd_mgmt_rsp rsp;
+  ubbd_unmap_options opts = { 0 };
+  struct stat sb;
+  int r;
+
+  if (!boost::starts_with(device_name, "/dev/ubbd")) {
+    std::cerr << "rbd: ubbd unmap requires device path (/dev/ubbdX)" << std::endl;
+    return -EINVAL;
+  }
+
+  if (stat(device_name.c_str(), &sb) < 0 || !S_ISBLK(sb.st_mode)) {
+    std::cerr << "rbd: '" << device_name << "' is not a block device" << std::endl;
+    return -EINVAL;
+  }
+
+  device_name.erase(0, 9);
+  opts.ubbdid = stoi(device_name);
+
+  if (vm.count("options")) {
+    for (auto &options : vm["options"].as<std::vector<std::string>>()) {
+      r = parse_unmap_options(options, &opts);
+      if (r < 0) {
+        std::cerr << "rbd: couldn't parse ubbd unmap options" << std::endl;
+        return r;
+      }
+    }
+  }
+
+  r = ubbd_unmap(&opts, &rsp);
+  if (r < 0) {
+    std::cerr << "rbd: ubbd_unmap failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+#else
+  std::cerr << "rbd: ubbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+int execute_attach(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+#if defined(WITH_RBD_UBBD)
+  std::cerr << "rbd: ubbd does not support attach" << std::endl;
+#else
+  std::cerr << "rbd: ubbd device is not supported" << std::endl;
+#endif
+  return -EOPNOTSUPP;
+}
+
+int execute_detach(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+#if defined(WITH_RBD_UBBD)
+  std::cerr << "rbd: ubbd does not support detach" << std::endl;
+#else
+  std::cerr << "rbd: ubbd device is not supported" << std::endl;
+#endif
+  return -EOPNOTSUPP;
+}
+
+} // namespace ubbd
+} // namespace action
+} // namespace rbd