*
*/
+#include "acconfig.h"
#include "include/int_types.h"
#include <libgen.h>
#include "common/Formatter.h"
#include "common/Preforker.h"
+#include "common/SubProcess.h"
#include "common/TextTable.h"
#include "common/ceph_argparse.h"
#include "common/config.h"
int timeout = -1;
bool exclusive = false;
+ bool quiesce = false;
bool readonly = false;
bool set_max_part = false;
bool try_netlink = false;
std::string imgname;
std::string snapname;
std::string devpath;
+ std::string quiesce_hook = CMAKE_INSTALL_LIBEXECDIR "/rbd-nbd/rbd-nbd_quiesce";
std::string format;
bool pretty_format = false;
<< " [options] list-mapped List mapped nbd devices\n"
<< "Map options:\n"
<< " --device <device path> Specify nbd device path (/dev/nbd{num})\n"
- << " --read-only Map read-only\n"
- << " --nbds_max <limit> Override for module param nbds_max\n"
- << " --max_part <limit> Override for module param max_part\n"
<< " --exclusive Forbid writes by other clients\n"
+ << " --max_part <limit> Override for module param max_part\n"
+ << " --nbds_max <limit> Override for module param nbds_max\n"
+ << " --quiesce Use quiesce callbacks\n"
+ << " --quiesce_hook <path> Specify quiesce hook path\n"
+ << " (default: " << Config().quiesce_hook << ")\n"
+ << " --read-only Map read-only\n"
<< " --timeout <seconds> Set nbd request timeout\n"
<< " --try-netlink Use the nbd netlink interface\n"
<< "\n"
Command *command, Config *cfg);
static int netlink_resize(int nbd_index, uint64_t size);
+static void run_quiesce_hook(const std::string &quiesce_hook,
+ const std::string &devpath,
+ const std::string &command);
+
class NBDServer
{
private:
int fd;
librbd::Image ℑ
+ Config *cfg;
public:
- NBDServer(int _fd, librbd::Image& _image)
- : fd(_fd)
- , image(_image)
+ NBDServer(int fd, librbd::Image& image, Config *cfg)
+ : fd(fd)
+ , image(image)
+ , cfg(cfg)
, reader_thread(*this, &NBDServer::reader_entry)
, writer_thread(*this, &NBDServer::writer_entry)
+ , quiesce_thread(*this, &NBDServer::quiesce_entry)
, started(false)
{}
dout(20) << __func__ << ": terminated" << dendl;
}
+ bool wait_quiesce() {
+ dout(20) << __func__ << dendl;
+
+ std::unique_lock locker{lock};
+ cond.wait(locker, [this] { return quiesce || terminated; });
+
+ if (terminated) {
+ return false;
+ }
+
+ dout(20) << __func__ << ": got quiesce request" << dendl;
+ return true;
+ }
+
+ void wait_unquiesce() {
+ dout(20) << __func__ << dendl;
+
+ std::unique_lock locker{lock};
+ cond.wait(locker, [this] { return !quiesce || terminated; });
+
+ dout(20) << __func__ << ": got unquiesce request" << dendl;
+ }
+
+ void wait_inflight_io() {
+ uint64_t features = 0;
+ image.features(&features);
+ if ((features & RBD_FEATURE_EXCLUSIVE_LOCK) != 0) {
+ bool is_owner = false;
+ image.is_exclusive_lock_owner(&is_owner);
+ if (!is_owner) {
+ return;
+ }
+ }
+
+ dout(20) << __func__ << dendl;
+
+ int r = image.flush();
+ if (r < 0) {
+ derr << "flush failed: " << cpp_strerror(r) << dendl;
+ }
+ }
+
+ void quiesce_entry()
+ {
+ ceph_assert(cfg->quiesce);
+
+ while (wait_quiesce()) {
+
+ run_quiesce_hook(cfg->quiesce_hook, cfg->devpath, "quiesce");
+
+ wait_inflight_io();
+
+ image.quiesce_complete();
+
+ wait_unquiesce();
+
+ run_quiesce_hook(cfg->quiesce_hook, cfg->devpath, "unquiesce");
+ }
+ }
+
class ThreadHelper : public Thread
{
public:
server.shutdown();
return NULL;
}
- } reader_thread, writer_thread;
+ } reader_thread, writer_thread, quiesce_thread;
bool started;
+ bool quiesce;
+
public:
void start()
{
reader_thread.create("rbd_reader");
writer_thread.create("rbd_writer");
+ if (cfg->quiesce) {
+ quiesce_thread.create("rbd_quiesce");
+ }
}
}
disconnect_cond.wait(l);
}
+ void notify_quiesce() {
+ dout(10) << __func__ << dendl;
+
+ ceph_assert(cfg->quiesce);
+
+ std::unique_lock locker{lock};
+ ceph_assert(quiesce == false);
+ quiesce = true;
+ cond.notify_all();
+ }
+
+ void notify_unquiesce() {
+ dout(10) << __func__ << dendl;
+
+ ceph_assert(cfg->quiesce);
+
+ std::unique_lock locker{lock};
+ ceph_assert(quiesce == true);
+ quiesce = false;
+ cond.notify_all();
+ }
+
~NBDServer()
{
if (started) {
reader_thread.join();
writer_thread.join();
+ if (cfg->quiesce) {
+ quiesce_thread.join();
+ }
wait_clean();
return os;
}
+class NBDQuiesceWatchCtx : public librbd::QuiesceWatchCtx
+{
+public:
+ NBDQuiesceWatchCtx(NBDServer *server) : server(server) {
+ }
+
+ void handle_quiesce() override {
+ server->notify_quiesce();
+ }
+
+ void handle_unquiesce() override {
+ server->notify_unquiesce();
+ }
+
+private:
+ NBDServer *server;
+};
+
class NBDWatchCtx : public librbd::UpdateWatchCtx
{
private:
return 0;
}
+static void run_quiesce_hook(const std::string &quiesce_hook,
+ const std::string &devpath,
+ const std::string &command) {
+ dout(10) << __func__ << ": " << quiesce_hook << " " << devpath << " "
+ << command << dendl;
+
+ SubProcess hook(quiesce_hook.c_str(), SubProcess::CLOSE, SubProcess::PIPE,
+ SubProcess::PIPE);
+ hook.add_cmd_args(devpath.c_str(), command.c_str(), NULL);
+ bufferlist err;
+ int r = hook.spawn();
+ if (r != 0) {
+ err.append("subprocess spawn failed");
+ } else {
+ err.read_fd(hook.get_stderr(), 16384);
+ r = hook.join();
+ }
+ if (r != 0) {
+ derr << __func__ << ": " << quiesce_hook << " " << devpath << " "
+ << command << " failed: " << err.to_str() << dendl;
+ } else {
+ dout(10) << " succeeded: " << err.to_str() << dendl;
+ }
+}
+
static void handle_signal(int signum)
{
int ret;
}
}
-static NBDServer *start_server(int fd, librbd::Image& image)
+static NBDServer *start_server(int fd, librbd::Image& image, Config *cfg)
{
NBDServer *server;
- server = new NBDServer(fd, image);
+ server = new NBDServer(fd, image, cfg);
server->start();
init_async_signal_handler();
if (r < 0)
goto close_fd;
- server = start_server(fd[1], image);
+ server = start_server(fd[1], image, cfg);
use_netlink = cfg->try_netlink;
if (use_netlink) {
}
{
+ NBDQuiesceWatchCtx quiesce_watch_ctx(server);
+ uint64_t quiesce_watch_handle;
+ if (cfg->quiesce) {
+ r = image.quiesce_watch(&quiesce_watch_ctx, &quiesce_watch_handle);
+ if (r < 0) {
+ goto close_nbd;
+ }
+ }
+
uint64_t handle;
NBDWatchCtx watch_ctx(nbd, nbd_index, use_netlink, io_ctx, image,
run_server(forker, server, use_netlink);
+ if (cfg->quiesce) {
+ r = image.quiesce_unwatch(quiesce_watch_handle);
+ ceph_assert(r == 0);
+ }
+
r = image.update_unwatch(handle);
ceph_assert(r == 0);
}
return -EINVAL;
}
cfg->set_max_part = true;
+ } else if (ceph_argparse_flag(args, i, "--quiesce", (char *)NULL)) {
+ cfg->quiesce = true;
+ } else if (ceph_argparse_witharg(args, i, &cfg->quiesce_hook,
+ "--quiesce-hook", (char *)NULL)) {
} else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) {
cfg->readonly = true;
} else if (ceph_argparse_flag(args, i, "--exclusive", (char *)NULL)) {
projects / ceph.git / commitdiff