#include <stddef.h>
#include <errno.h>
#include <fcntl.h>
+#include <poll.h>
#include <string.h>
#include <sys/types.h>
#include <unistd.h>
#include <libnl3/netlink/genl/ctrl.h>
#include <libnl3/netlink/genl/mngt.h>
+#if __has_include(<filesystem>)
+#include <filesystem>
+namespace fs = std::filesystem;
+#else
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#endif
#include <fstream>
#include <iostream>
#include <memory>
#include <regex>
#include <boost/algorithm/string/predicate.hpp>
+#include <boost/lexical_cast.hpp>
#include "common/Formatter.h"
#include "common/Preforker.h"
#undef dout_prefix
#define dout_prefix *_dout << "rbd-nbd: "
+enum Command {
+ None,
+ Map,
+ Unmap,
+ Attach,
+ Detach,
+ List
+};
+
struct Config {
int nbds_max = 0;
int max_part = 255;
std::string format;
bool pretty_format = false;
+
+ Command command = None;
+ int pid = 0;
+
+ std::string image_spec() const {
+ std::string spec = poolname + "/";
+
+ if (!nsname.empty()) {
+ spec += "/" + nsname;
+ }
+ spec += imgname;
+
+ if (!snapname.empty()) {
+ spec += "@" + snapname;
+ }
+
+ return spec;
+ }
};
static void usage()
{
- std::cout << "Usage: rbd-nbd [options] map <image-or-snap-spec> Map an image to nbd device\n"
- << " unmap <device|image-or-snap-spec> Unmap nbd device\n"
- << " [options] list-mapped List mapped nbd devices\n"
- << "Map options:\n"
+ std::cout << "Usage: rbd-nbd [options] map <image-or-snap-spec> Map image to nbd device\n"
+ << " detach <device|image-or-snap-spec> Detach image from nbd device\n"
+ << " [options] attach <image-or-snap-spec> Attach image to nbd device\n"
+ << " unmap <device|image-or-snap-spec> Unmap nbd device\n"
+ << " [options] list-mapped List mapped nbd devices\n"
+ << "Map and attach options:\n"
<< " --device <device path> Specify nbd device path (/dev/nbd{num})\n"
<< " --exclusive Forbid writes by other clients\n"
<< " --max_part <limit> Override for module param max_part\n"
static int nbd = -1;
static int nbd_index = -1;
-
-enum Command {
- None,
- Connect,
- Disconnect,
- List
-};
-
-static Command cmd = None;
+static bool nbd_terminate = false;
#define RBD_NBD_BLKSIZE 512UL
#define htonll(a) ntohll(a)
static int parse_args(vector<const char*>& args, std::ostream *err_msg,
- Command *command, Config *cfg);
+ Config *cfg);
static int netlink_resize(int nbd_index, uint64_t size);
static int run_quiesce_hook(const std::string &quiesce_hook,
}
}
+ Config *get_cfg() const {
+ return cfg;
+ }
+
private:
ceph::mutex disconnect_lock =
ceph::make_mutex("NBDServer::DisconnectLocker");
void reader_entry()
{
+ struct pollfd poll_fds[1];
+ memset(poll_fds, 0, sizeof(struct pollfd));
+ poll_fds[0].fd = fd;
+ poll_fds[0].events = POLLIN;
+
while (!terminated) {
std::unique_ptr<IOContext> ctx(new IOContext());
ctx->server = this;
dout(20) << __func__ << ": waiting for nbd request" << dendl;
- int r = safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request));
+ // We can not block on read because we have to check periodically
+ // nbd_terminate flag set by the signal handler.
+ // So to prevent blocking use poll with timeout to ensure we
+ // have some data comes before calling read.
+
+ int r = poll(poll_fds, 1, 1000);
+ if (r == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ r = -errno;
+ derr << "failed to poll nbd: " << cpp_strerror(r) << dendl;
+ goto signal;
+ }
+
+ if (nbd_terminate) {
+ dout(0) << "terminate received" << dendl;
+ goto signal;
+ }
+
+ if ((poll_fds[0].revents & POLLIN) == 0) {
+ dout(20) << __func__ << ": poll timed out" << dendl;
+ continue;
+ }
+
+ r = safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request));
if (r < 0) {
derr << "failed to read nbd request header: " << cpp_strerror(r)
<< dendl;
case NBD_CMD_TRIM:
os << " TRIM ";
break;
+ case NBD_CMD_DISC:
+ os << " DISC ";
+ break;
default:
os << " UNKNOWN(" << ctx.command << ") ";
break;
class NBDListIterator {
public:
- bool get(int *pid, Config *cfg) {
+ bool get(Config *cfg) {
while (true) {
std::string nbd_path = "/sys/block/nbd" + stringify(m_index);
if(access(nbd_path.c_str(), F_OK) != 0) {
*cfg = Config();
cfg->devpath = "/dev/nbd" + stringify(m_index++);
+ int pid;
std::ifstream ifs;
ifs.open(nbd_path + "/pid", std::ifstream::in);
if (!ifs.is_open()) {
continue;
}
- ifs >> *pid;
-
- int r = get_mapped_info(*pid, cfg);
- if (r < 0) {
- continue;
- }
+ ifs >> pid;
+
+ // If the rbd-nbd is re-attached the pid may store garbage
+ // here. We are sure this is the case when it is negative or
+ // zero. Then we just try to find the attached process scanning
+ // /proc fs. If it is positive we check the process with this
+ // pid first and if it is not rbd-nbd fallback to searching the
+ // attached process.
+ do {
+ if (pid <= 0) {
+ pid = find_attached(cfg->devpath);
+ if (pid <= 0) {
+ break;
+ }
+ }
- return true;
+ if (get_mapped_info(pid, cfg) >= 0) {
+ return true;
+ }
+ pid = -1;
+ } while (true);
}
}
private:
int m_index = 0;
+ std::map<int, Config> m_mapped_info_cache;
int get_mapped_info(int pid, Config *cfg) {
+ auto it = m_mapped_info_cache.find(pid);
+ if (it != m_mapped_info_cache.end()) {
+ if (it->second.devpath.empty()) {
+ return -EINVAL;
+ }
+ *cfg = it->second;
+ return 0;
+ }
+
+ m_mapped_info_cache[pid] = {};
+
int r;
std::string path = "/proc/" + stringify(pid) + "/cmdline";
std::ifstream ifs;
return -1;
ifs >> cmdline;
+ if (cmdline.empty()) {
+ return -EINVAL;
+ }
+
for (unsigned i = 0; i < cmdline.size(); i++) {
char *arg = &cmdline[i];
if (i == 0) {
}
std::ostringstream err_msg;
- Command command;
- r = parse_args(args, &err_msg, &command, cfg);
+ r = parse_args(args, &err_msg, cfg);
if (r < 0) {
return r;
}
- if (command != Connect) {
+ if (cfg->command != Map && cfg->command != Attach) {
return -ENOENT;
}
+ cfg->pid = pid;
+
+ m_mapped_info_cache.erase(pid);
+ if (!cfg->devpath.empty()) {
+ m_mapped_info_cache[pid] = *cfg;
+ }
+
return 0;
}
+
+ int find_attached(const std::string &devpath) {
+ for (auto &entry : fs::directory_iterator("/proc")) {
+ if (!fs::is_directory(entry.status())) {
+ continue;
+ }
+
+ int pid;
+ try {
+ pid = boost::lexical_cast<uint64_t>(entry.path().filename().c_str());
+ } catch (boost::bad_lexical_cast&) {
+ continue;
+ }
+
+ Config cfg;
+ if (get_mapped_info(pid, &cfg) >=0 && cfg.devpath == devpath &&
+ cfg.command == Attach) {
+ return cfg.pid;
+ }
+ }
+
+ return -1;
+ }
};
static int load_module(Config *cfg)
r = ioctl(nbd, NBD_SET_BLKSIZE, RBD_NBD_BLKSIZE);
if (r < 0) {
r = -errno;
+ cerr << "rbd-nbd: NBD_SET_BLKSIZE failed" << std::endl;
goto close_nbd;
}
r = ioctl(nbd, NBD_SET_SIZE, size);
if (r < 0) {
+ cerr << "rbd-nbd: NBD_SET_SIZE failed" << std::endl;
r = -errno;
goto close_nbd;
}
}
static int netlink_connect(Config *cfg, struct nl_sock *sock, int nl_id, int fd,
- uint64_t size, uint64_t flags)
+ uint64_t size, uint64_t flags, bool reconnect)
{
struct nlattr *sock_attr;
struct nlattr *sock_opt;
struct nl_msg *msg;
int ret;
- nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, netlink_connect_cb, cfg);
+ if (reconnect) {
+ dout(10) << "netlink try reconnect for " << cfg->devpath << dendl;
+
+ nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, genl_handle_msg, NULL);
+ } else {
+ nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, netlink_connect_cb,
+ cfg);
+ }
msg = nlmsg_alloc();
if (!msg) {
return -ENOMEM;
}
- if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0, NBD_CMD_CONNECT,
- 0)) {
+ if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0,
+ reconnect ? NBD_CMD_RECONFIGURE : NBD_CMD_CONNECT, 0)) {
cerr << "rbd-nbd: Could not setup message." << std::endl;
goto free_msg;
}
goto free_msg;
NLA_PUT_U32(msg, NBD_ATTR_INDEX, ret);
+ if (reconnect) {
+ nbd_index = ret;
+ }
}
if (cfg->timeout >= 0)
NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, RBD_NBD_BLKSIZE);
NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags);
+ NLA_PUT_U64(msg, NBD_ATTR_DEAD_CONN_TIMEOUT,
+ cfg->timeout >= 0 ? cfg->timeout : 30);
sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS);
if (!sock_attr) {
return -EIO;
}
-static int try_netlink_setup(Config *cfg, int fd, uint64_t size, uint64_t flags)
+static int try_netlink_setup(Config *cfg, int fd, uint64_t size, uint64_t flags,
+ bool reconnect)
{
struct nl_sock *sock;
int nl_id, ret;
dout(10) << "netlink interface supported." << dendl;
- ret = netlink_connect(cfg, sock, nl_id, fd, size, flags);
+ ret = netlink_connect(cfg, sock, nl_id, fd, size, flags, reconnect);
netlink_cleanup(sock);
if (ret != 0)
static void handle_signal(int signum)
{
- int ret;
-
ceph_assert(signum == SIGINT || signum == SIGTERM);
derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
- if (nbd < 0 || nbd_index < 0) {
- dout(20) << __func__ << ": " << "disconnect not needed." << dendl;
- return;
- }
-
- dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
- ret = netlink_disconnect(nbd_index);
- if (ret == 1)
- ret = ioctl(nbd, NBD_DISCONNECT);
-
- if (ret != 0) {
- derr << "rbd-nbd: disconnect failed. Error: " << ret << dendl;
- } else {
- dout(20) << __func__ << ": " << "disconnected" << dendl;
- }
+ dout(20) << __func__ << ": " << "setting terminate flag" << dendl;
+ nbd_terminate = true;
}
static NBDServer *start_server(int fd, librbd::Image& image, Config *cfg)
shutdown_async_signal_handler();
}
-static int do_map(int argc, const char *argv[], Config *cfg)
+static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect)
{
int r;
server = start_server(fd[1], image, cfg);
- use_netlink = cfg->try_netlink;
+ use_netlink = cfg->try_netlink || reconnect;
if (use_netlink) {
- r = try_netlink_setup(cfg, fd[0], size, flags);
+ r = try_netlink_setup(cfg, fd[0], size, flags, reconnect);
if (r < 0) {
goto free_server;
} else if (r == 1) {
return r;
}
+static int do_detach(Config *cfg)
+{
+ int r = kill(cfg->pid, SIGTERM);
+ if (r != 0) {
+ r = -errno;
+ cerr << "rbd-nbd: failed to terminate " << cfg->pid << ": "
+ << cpp_strerror(r) << std::endl;
+ return r;
+ }
+
+ return 0;
+}
+
static int do_unmap(Config *cfg)
{
int r, nbd;
tbl.define_column("device", TextTable::LEFT, TextTable::LEFT);
}
- int pid;
Config cfg;
NBDListIterator it;
- while (it.get(&pid, &cfg)) {
+ while (it.get(&cfg)) {
if (f) {
f->open_object_section("device");
- f->dump_int("id", pid);
+ f->dump_int("id", cfg.pid);
f->dump_string("pool", cfg.poolname);
f->dump_string("namespace", cfg.nsname);
f->dump_string("image", cfg.imgname);
if (cfg.snapname.empty()) {
cfg.snapname = "-";
}
- tbl << pid << cfg.poolname << cfg.nsname << cfg.imgname << cfg.snapname
- << cfg.devpath << TextTable::endrow;
+ tbl << cfg.pid << cfg.poolname << cfg.nsname << cfg.imgname
+ << cfg.snapname << cfg.devpath << TextTable::endrow;
}
}
return 0;
}
-static bool find_mapped_dev_by_spec(Config *cfg) {
- int pid;
+static bool find_mapped_dev_by_spec(Config *cfg, int skip_pid=-1) {
Config c;
NBDListIterator it;
- while (it.get(&pid, &c)) {
- if (c.poolname == cfg->poolname && c.nsname == cfg->nsname &&
+ while (it.get(&c)) {
+ if (c.pid != skip_pid &&
+ c.poolname == cfg->poolname && c.nsname == cfg->nsname &&
c.imgname == cfg->imgname && c.snapname == cfg->snapname) {
*cfg = c;
return true;
static int parse_args(vector<const char*>& args, std::ostream *err_msg,
- Command *command, Config *cfg) {
+ Config *cfg) {
std::string conf_file_list;
std::string cluster;
CephInitParameters iparams = ceph_argparse_early_args(
Command cmd = None;
if (args.begin() != args.end()) {
if (strcmp(*args.begin(), "map") == 0) {
- cmd = Connect;
+ cmd = Map;
} else if (strcmp(*args.begin(), "unmap") == 0) {
- cmd = Disconnect;
+ cmd = Unmap;
+ } else if (strcmp(*args.begin(), "attach") == 0) {
+ cmd = Attach;
+ } else if (strcmp(*args.begin(), "detach") == 0) {
+ cmd = Detach;
} else if (strcmp(*args.begin(), "list-mapped") == 0) {
cmd = List;
} else {
}
switch (cmd) {
- case Connect:
+ case Attach:
+ if (cfg->devpath.empty()) {
+ *err_msg << "rbd-nbd: must specify device to attach";
+ return -EINVAL;
+ }
+ [[fallthrough]];
+ case Map:
if (args.begin() == args.end()) {
*err_msg << "rbd-nbd: must specify image-or-snap-spec";
return -EINVAL;
}
args.erase(args.begin());
break;
- case Disconnect:
+ case Detach:
+ case Unmap:
if (args.begin() == args.end()) {
*err_msg << "rbd-nbd: must specify nbd device or image-or-snap-spec";
return -EINVAL;
if (parse_imgpath(*args.begin(), cfg, err_msg) < 0) {
return -EINVAL;
}
- if (!find_mapped_dev_by_spec(cfg)) {
- *err_msg << "rbd-nbd: " << *args.begin() << " is not mapped";
- return -ENOENT;
- }
}
args.erase(args.begin());
break;
return -EINVAL;
}
- *command = cmd;
+ cfg->command = cmd;
return 0;
}
argv_to_vec(argc, argv, args);
std::ostringstream err_msg;
- r = parse_args(args, &err_msg, &cmd, &cfg);
+ r = parse_args(args, &err_msg, &cfg);
if (r == HELP_INFO) {
usage();
return 0;
return r;
}
- switch (cmd) {
- case Connect:
+ switch (cfg.command) {
+ case Attach:
+ ceph_assert(!cfg.devpath.empty());
+ if (find_mapped_dev_by_spec(&cfg, getpid())) {
+ cerr << "rbd-nbd: " << cfg.devpath << " has process " << cfg.pid
+ << " connected" << std::endl;
+ return -EBUSY;
+ }
+ [[fallthrough]];
+ case Map:
if (cfg.imgname.empty()) {
cerr << "rbd-nbd: image name was not specified" << std::endl;
return -EINVAL;
}
- r = do_map(argc, argv, &cfg);
+ r = do_map(argc, argv, &cfg, cfg.command == Attach);
if (r < 0)
return -EINVAL;
break;
- case Disconnect:
+ case Detach:
+ if (cfg.devpath.empty() && !find_mapped_dev_by_spec(&cfg)) {
+ cerr << "rbd-nbd: " << cfg.image_spec() << " is not mapped"
+ << std::endl;
+ return -ENOENT;
+ }
+ r = do_detach(&cfg);
+ if (r < 0)
+ return -EINVAL;
+ break;
+ case Unmap:
+ if (cfg.devpath.empty() && !find_mapped_dev_by_spec(&cfg)) {
+ cerr << "rbd-nbd: " << cfg.image_spec() << " is not mapped"
+ << std::endl;
+ return -ENOENT;
+ }
r = do_unmap(&cfg);
if (r < 0)
return -EINVAL;
projects / ceph.git / commitdiff