From: Mykola Golub Date: Mon, 27 Apr 2020 06:32:19 +0000 (+0100) Subject: rbd-nbd: add quiesce/unquiesce hooks X-Git-Tag: v16.1.0~2324^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d01295494d7523751907c347ae7bccc85c53c6e;p=ceph.git rbd-nbd: add quiesce/unquiesce hooks Signed-off-by: Mykola Golub --- diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index ea550c81e3a..e26671d3a00 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -351,4 +351,7 @@ /* Shared library extension, such as .so, .dll or .dylib */ #cmakedefine CMAKE_SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@" +/* libexec directory path */ +#cmakedefine CMAKE_INSTALL_LIBEXECDIR "@CMAKE_INSTALL_LIBEXECDIR@" + #endif /* CONFIG_H */ diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index fc2bc57749c..18d40de6724 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -582,8 +582,8 @@ rbd help device map usage: rbd device map [--device-type ] [--pool ] [--namespace ] [--image ] - [--snap ] [--read-only] [--exclusive] - [--options ] + [--snap ] [--read-only] [--exclusive] [--quiesce] + [--quiesce-hook ] [--options ] Map an image to a block device. @@ -602,6 +602,8 @@ --snap arg snapshot name --read-only map read-only --exclusive disable automatic exclusive lock transitions + --quiesce use quiesce callbacks + --quiesce-hook arg quiesce hook path -o [ --options ] arg device specific options rbd help device unmap diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc index 3fdf2ef50aa..94864d04c63 100644 --- a/src/tools/rbd/action/Device.cc +++ b/src/tools/rbd/action/Device.cc @@ -136,7 +136,9 @@ void get_map_arguments(po::options_description *positional, at::ARGUMENT_MODIFIER_NONE); options->add_options() ("read-only", po::bool_switch(), "map read-only") - ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions"); + ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions") + ("quiesce", po::bool_switch(), "use quiesce callbacks") + ("quiesce-hook", po::value(), "quiesce hook path"); add_device_specific_options(options); } diff --git a/src/tools/rbd/action/Ggate.cc b/src/tools/rbd/action/Ggate.cc index 61f77be2db3..74cd57dd5a2 100644 --- a/src/tools/rbd/action/Ggate.cc +++ b/src/tools/rbd/action/Ggate.cc @@ -130,6 +130,10 @@ int execute_map(const po::variables_map &vm, } args.push_back(img); + if (vm["quiesce"].as()) { + std::cerr << "rbd: warning: quiesce is not supported" << std::endl; + } + if (vm["read-only"].as()) { args.push_back("--read-only"); } @@ -138,6 +142,10 @@ int execute_map(const po::variables_map &vm, args.push_back("--exclusive"); } + if (vm.count("quiesce-hook")) { + std::cerr << "rbd: warning: quiesce-hook is not supported" << std::endl; + } + if (vm.count("options")) { r = parse_options(vm["options"].as>(), &args); if (r < 0) { diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc index 3ebb5bc1de4..3b6bfd10523 100644 --- a/src/tools/rbd/action/Kernel.cc +++ b/src/tools/rbd/action/Kernel.cc @@ -434,6 +434,9 @@ int execute_map(const po::variables_map &vm, return r; } + if (vm["quiesce"].as()) { + std::cerr << "rbd: warning: quiesce is not supported" << std::endl; + } if (vm["read-only"].as()) { put_map_option("rw", "ro"); } @@ -441,6 +444,10 @@ int execute_map(const po::variables_map &vm, put_map_option("exclusive", "exclusive"); } + if (vm.count("quiesce-hook")) { + std::cerr << "rbd: warning: quiesce-hook is not supported" << std::endl; + } + // parse default options first so they can be overwritten by cli options r = parse_map_options( g_conf().get_val("rbd_default_map_options")); diff --git a/src/tools/rbd/action/Nbd.cc b/src/tools/rbd/action/Nbd.cc index 5c55adea4e7..ab62e9a7bf8 100644 --- a/src/tools/rbd/action/Nbd.cc +++ b/src/tools/rbd/action/Nbd.cc @@ -134,6 +134,10 @@ int execute_map(const po::variables_map &vm, } args.push_back(img); + if (vm["quiesce"].as()) { + args.push_back("--quiesce"); + } + if (vm["read-only"].as()) { args.push_back("--read-only"); } @@ -142,6 +146,11 @@ int execute_map(const po::variables_map &vm, args.push_back("--exclusive"); } + if (vm.count("quiesce-hook")) { + args.push_back("--quiesce-hook"); + args.push_back(vm["quiesce-hook"].as()); + } + if (vm.count("options")) { r = parse_options(vm["options"].as>(), &args); if (r < 0) { diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 77d8e82e299..a9af90922c9 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -16,6 +16,7 @@ * */ +#include "acconfig.h" #include "include/int_types.h" #include @@ -46,6 +47,7 @@ #include "common/Formatter.h" #include "common/Preforker.h" +#include "common/SubProcess.h" #include "common/TextTable.h" #include "common/ceph_argparse.h" #include "common/config.h" @@ -76,6 +78,7 @@ struct Config { int timeout = -1; bool exclusive = false; + bool quiesce = false; bool readonly = false; bool set_max_part = false; bool try_netlink = false; @@ -85,6 +88,7 @@ struct Config { std::string imgname; std::string snapname; std::string devpath; + std::string quiesce_hook = CMAKE_INSTALL_LIBEXECDIR "/rbd-nbd/rbd-nbd_quiesce"; std::string format; bool pretty_format = false; @@ -97,10 +101,13 @@ static void usage() << " [options] list-mapped List mapped nbd devices\n" << "Map options:\n" << " --device Specify nbd device path (/dev/nbd{num})\n" - << " --read-only Map read-only\n" - << " --nbds_max Override for module param nbds_max\n" - << " --max_part Override for module param max_part\n" << " --exclusive Forbid writes by other clients\n" + << " --max_part Override for module param max_part\n" + << " --nbds_max Override for module param nbds_max\n" + << " --quiesce Use quiesce callbacks\n" + << " --quiesce_hook Specify quiesce hook path\n" + << " (default: " << Config().quiesce_hook << ")\n" + << " --read-only Map read-only\n" << " --timeout Set nbd request timeout\n" << " --try-netlink Use the nbd netlink interface\n" << "\n" @@ -141,18 +148,25 @@ static int parse_args(vector& args, std::ostream *err_msg, Command *command, Config *cfg); static int netlink_resize(int nbd_index, uint64_t size); +static void run_quiesce_hook(const std::string &quiesce_hook, + const std::string &devpath, + const std::string &command); + class NBDServer { private: int fd; librbd::Image ℑ + Config *cfg; public: - NBDServer(int _fd, librbd::Image& _image) - : fd(_fd) - , image(_image) + NBDServer(int fd, librbd::Image& image, Config *cfg) + : fd(fd) + , image(image) + , cfg(cfg) , reader_thread(*this, &NBDServer::reader_entry) , writer_thread(*this, &NBDServer::writer_entry) + , quiesce_thread(*this, &NBDServer::quiesce_entry) , started(false) {} @@ -380,6 +394,66 @@ signal: dout(20) << __func__ << ": terminated" << dendl; } + bool wait_quiesce() { + dout(20) << __func__ << dendl; + + std::unique_lock locker{lock}; + cond.wait(locker, [this] { return quiesce || terminated; }); + + if (terminated) { + return false; + } + + dout(20) << __func__ << ": got quiesce request" << dendl; + return true; + } + + void wait_unquiesce() { + dout(20) << __func__ << dendl; + + std::unique_lock locker{lock}; + cond.wait(locker, [this] { return !quiesce || terminated; }); + + dout(20) << __func__ << ": got unquiesce request" << dendl; + } + + void wait_inflight_io() { + uint64_t features = 0; + image.features(&features); + if ((features & RBD_FEATURE_EXCLUSIVE_LOCK) != 0) { + bool is_owner = false; + image.is_exclusive_lock_owner(&is_owner); + if (!is_owner) { + return; + } + } + + dout(20) << __func__ << dendl; + + int r = image.flush(); + if (r < 0) { + derr << "flush failed: " << cpp_strerror(r) << dendl; + } + } + + void quiesce_entry() + { + ceph_assert(cfg->quiesce); + + while (wait_quiesce()) { + + run_quiesce_hook(cfg->quiesce_hook, cfg->devpath, "quiesce"); + + wait_inflight_io(); + + image.quiesce_complete(); + + wait_unquiesce(); + + run_quiesce_hook(cfg->quiesce_hook, cfg->devpath, "unquiesce"); + } + } + class ThreadHelper : public Thread { public: @@ -399,9 +473,11 @@ signal: server.shutdown(); return NULL; } - } reader_thread, writer_thread; + } reader_thread, writer_thread, quiesce_thread; bool started; + bool quiesce; + public: void start() { @@ -412,6 +488,9 @@ public: reader_thread.create("rbd_reader"); writer_thread.create("rbd_writer"); + if (cfg->quiesce) { + quiesce_thread.create("rbd_quiesce"); + } } } @@ -424,6 +503,28 @@ public: disconnect_cond.wait(l); } + void notify_quiesce() { + dout(10) << __func__ << dendl; + + ceph_assert(cfg->quiesce); + + std::unique_lock locker{lock}; + ceph_assert(quiesce == false); + quiesce = true; + cond.notify_all(); + } + + void notify_unquiesce() { + dout(10) << __func__ << dendl; + + ceph_assert(cfg->quiesce); + + std::unique_lock locker{lock}; + ceph_assert(quiesce == true); + quiesce = false; + cond.notify_all(); + } + ~NBDServer() { if (started) { @@ -433,6 +534,9 @@ public: reader_thread.join(); writer_thread.join(); + if (cfg->quiesce) { + quiesce_thread.join(); + } wait_clean(); @@ -470,6 +574,24 @@ std::ostream &operator<<(std::ostream &os, const NBDServer::IOContext &ctx) { return os; } +class NBDQuiesceWatchCtx : public librbd::QuiesceWatchCtx +{ +public: + NBDQuiesceWatchCtx(NBDServer *server) : server(server) { + } + + void handle_quiesce() override { + server->notify_quiesce(); + } + + void handle_unquiesce() override { + server->notify_unquiesce(); + } + +private: + NBDServer *server; +}; + class NBDWatchCtx : public librbd::UpdateWatchCtx { private: @@ -1047,6 +1169,31 @@ static int try_netlink_setup(Config *cfg, int fd, uint64_t size, uint64_t flags) return 0; } +static void run_quiesce_hook(const std::string &quiesce_hook, + const std::string &devpath, + const std::string &command) { + dout(10) << __func__ << ": " << quiesce_hook << " " << devpath << " " + << command << dendl; + + SubProcess hook(quiesce_hook.c_str(), SubProcess::CLOSE, SubProcess::PIPE, + SubProcess::PIPE); + hook.add_cmd_args(devpath.c_str(), command.c_str(), NULL); + bufferlist err; + int r = hook.spawn(); + if (r != 0) { + err.append("subprocess spawn failed"); + } else { + err.read_fd(hook.get_stderr(), 16384); + r = hook.join(); + } + if (r != 0) { + derr << __func__ << ": " << quiesce_hook << " " << devpath << " " + << command << " failed: " << err.to_str() << dendl; + } else { + dout(10) << " succeeded: " << err.to_str() << dendl; + } +} + static void handle_signal(int signum) { int ret; @@ -1071,11 +1218,11 @@ static void handle_signal(int signum) } } -static NBDServer *start_server(int fd, librbd::Image& image) +static NBDServer *start_server(int fd, librbd::Image& image, Config *cfg) { NBDServer *server; - server = new NBDServer(fd, image); + server = new NBDServer(fd, image, cfg); server->start(); init_async_signal_handler(); @@ -1221,7 +1368,7 @@ static int do_map(int argc, const char *argv[], Config *cfg) if (r < 0) goto close_fd; - server = start_server(fd[1], image); + server = start_server(fd[1], image, cfg); use_netlink = cfg->try_netlink; if (use_netlink) { @@ -1250,6 +1397,15 @@ static int do_map(int argc, const char *argv[], Config *cfg) } { + NBDQuiesceWatchCtx quiesce_watch_ctx(server); + uint64_t quiesce_watch_handle; + if (cfg->quiesce) { + r = image.quiesce_watch(&quiesce_watch_ctx, &quiesce_watch_handle); + if (r < 0) { + goto close_nbd; + } + } + uint64_t handle; NBDWatchCtx watch_ctx(nbd, nbd_index, use_netlink, io_ctx, image, @@ -1262,6 +1418,11 @@ static int do_map(int argc, const char *argv[], Config *cfg) run_server(forker, server, use_netlink); + if (cfg->quiesce) { + r = image.quiesce_unwatch(quiesce_watch_handle); + ceph_assert(r == 0); + } + r = image.update_unwatch(handle); ceph_assert(r == 0); } @@ -1466,6 +1627,10 @@ static int parse_args(vector& args, std::ostream *err_msg, return -EINVAL; } cfg->set_max_part = true; + } else if (ceph_argparse_flag(args, i, "--quiesce", (char *)NULL)) { + cfg->quiesce = true; + } else if (ceph_argparse_witharg(args, i, &cfg->quiesce_hook, + "--quiesce-hook", (char *)NULL)) { } else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) { cfg->readonly = true; } else if (ceph_argparse_flag(args, i, "--exclusive", (char *)NULL)) {