From 0ba3960c608708132f5980a212fa08691715b85a Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Thu, 17 Apr 2014 19:03:24 +0400 Subject: [PATCH] rbd: switch to libkrbd for 'rbd {map,showmapped,unmap}' operations Thanks to libkrbd, 'rbd map' now outputs the device node it mapped to to stdout: $ sudo rbd map foo /dev/rbd0 This will allow us to get rid of a lot of ad-hoc poll/sleep code in our qa scripts. Signed-off-by: Ilya Dryomov --- src/Makefile.am | 2 +- src/rbd.cc | 433 ++++-------------------------------------------- 2 files changed, 34 insertions(+), 401 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index bc498f8094831..ecc7aeaf6dda0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -82,7 +82,7 @@ noinst_LTLIBRARIES += libkrbd.la endif LINUX rbd_SOURCES = rbd.cc -rbd_LDADD = $(LIBSECRET) $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) -lblkid +rbd_LDADD = $(LIBKRBD) $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) if LINUX bin_PROGRAMS += rbd endif #LINUX diff --git a/src/rbd.cc b/src/rbd.cc index 8a99bbd9f9b85..9c31725b51131 100644 --- a/src/rbd.cc +++ b/src/rbd.cc @@ -11,16 +11,14 @@ #include "include/int_types.h" #include "mon/MonClient.h" -#include "mon/MonMap.h" #include "common/config.h" -#include "auth/KeyRing.h" #include "common/errno.h" #include "common/ceph_argparse.h" #include "common/strtol.h" #include "global/global_init.h" #include "common/safe_io.h" -#include "common/secret.h" +#include "include/krbd.h" #include "include/stringify.h" #include "include/rados/librados.hpp" #include "include/rbd/librbd.hpp" @@ -32,7 +30,6 @@ #include "common/blkdev.h" #include -#include #include #include #include @@ -57,8 +54,6 @@ #include #endif -#include - #define MAX_SECRET_LEN 1000 #define MAX_POOL_NAME_SIZE 128 @@ -1656,58 +1651,17 @@ static int do_watch(librados::IoCtx& pp, const char *imgname) return 0; } -static int do_kernel_add(const char *poolname, const char *imgname, +static int do_kernel_map(const char *poolname, const char *imgname, const char *snapname) { - MonMap monmap; - int r = monmap.build_initial(g_ceph_context, cerr); - if (r < 0) - return r; - - map::const_iterator it = monmap.mon_addr.begin(); + struct krbd_ctx *krbd; ostringstream oss; - for (size_t i = 0; i < monmap.mon_addr.size(); ++i, ++it) { - oss << it->second.addr; - if (i + 1 < monmap.mon_addr.size()) - oss << ","; - } - - const char *user = g_conf->name.get_id().c_str(); - oss << " name=" << user; - - char key_name[strlen(user) + strlen("client.") + 1]; - snprintf(key_name, sizeof(key_name), "client.%s", user); + char *devnode; + int r; - KeyRing keyring; - r = keyring.from_ceph_context(g_ceph_context); - if (r == -ENOENT && !(g_conf->keyfile.length() || - g_conf->key.length())) - r = 0; - if (r < 0) { - cerr << "rbd: failed to get secret: " << cpp_strerror(r) << std::endl; + r = krbd_create_from_context(g_ceph_context, &krbd); + if (r < 0) return r; - } - CryptoKey secret; - if (keyring.get_secret(g_conf->name, secret)) { - string secret_str; - secret.encode_base64(secret_str); - - r = set_kernel_secret(secret_str.c_str(), key_name); - if (r >= 0) { - if (r == 0) - cerr << "rbd: warning: secret has length 0" << std::endl; - oss << ",key=" << key_name; - } else if (r == -ENODEV || r == -ENOSYS) { - /* running against older kernel; fall back to secret= in options */ - oss << ",secret=" << secret_str; - } else { - cerr << "rbd: failed to add ceph secret key '" << key_name - << "' to kernel: " << cpp_strerror(r) << std::endl; - return r; - } - } else if (is_kernel_secret(key_name)) { - oss << ",key=" << key_name; - } for (map::const_iterator it = map_options.begin(); it != map_options.end(); @@ -1718,371 +1672,50 @@ static int do_kernel_add(const char *poolname, const char *imgname, if (it->first == "rw" && it->second == "rw") continue; - oss << "," << it->second; - } - - oss << " " << poolname << " " << imgname; - - if (snapname) { - oss << " " << snapname; - } - - // modprobe the rbd module if /sys/bus/rbd doesn't exist - struct stat sb; - if ((stat("/sys/bus/rbd", &sb) < 0) || (!S_ISDIR(sb.st_mode))) { - // turn on single-major device number allocation scheme if the - // kernel supports it - const char *cmd = "/sbin/modprobe rbd"; - r = system("/sbin/modinfo -F parm rbd | /bin/grep -q ^single_major:"); - if (r == 0) { - cmd = "/sbin/modprobe rbd single_major=Y"; - } else if (r < 0) { - cerr << "rbd: error executing modinfo as shell command!" << std::endl; - } - - r = system(cmd); - if (r) { - if (r < 0) - cerr << "rbd: error executing modprobe as shell command!" << std::endl; - else - cerr << "rbd: modprobe rbd failed! (" << r << ")" <second; } - // 'add' interface is deprecated, use 'add_single_major' if it's - // available - // - // ('add' and 'add_single_major' interfaces are identical, except - // that if rbd kernel module is new enough and is configured to use - // single-major scheme, 'add' is disabled in order to prevent old - // userspace from doing weird things at unmap time) - int fd = open("/sys/bus/rbd/add_single_major", O_WRONLY); - if (fd < 0) { - if (errno == ENOENT) { - fd = open("/sys/bus/rbd/add", O_WRONLY); - if (fd < 0) { - r = -errno; - if (r == -ENOENT) { - cerr << "rbd: /sys/bus/rbd/add does not exist!" << std::endl - << "Did you run 'modprobe rbd' or is your rbd module too old?" - << std::endl; - } - return r; - } - } else { - return -errno; - } - } + r = krbd_map(krbd, poolname, imgname, snapname, oss.str().c_str(), &devnode); + if (r < 0) + goto out; - string add = oss.str(); - r = safe_write(fd, add.c_str(), add.size()); - close(fd); - - // let udevadm do its job before we return - if (udevadm_settle) { - int r = system("/sbin/udevadm settle"); - if (r) { - if (r < 0) - cerr << "rbd: error executing udevadm as shell command!" << std::endl; - else - cerr << "rbd: '/sbin/udevadm settle' failed! (" << r << ")" < device_dir(opendir(devices_path), do_closedir); - if (!device_dir.get()) { - r = -errno; - cerr << "rbd: could not open " << devices_path << ": " - << cpp_strerror(-r) << std::endl; - return r; - } - - struct dirent *dent; - dent = readdir(device_dir.get()); - if (!dent) { - r = -errno; - cerr << "rbd: error reading " << devices_path << ": " - << cpp_strerror(-r) << std::endl; - return r; - } - - if (f) { - f->open_object_section("devices"); - } else { - tbl.define_column("id", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("image", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("device", TextTable::LEFT, TextTable::LEFT); - } - - do { - if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) - continue; - - char fn[PATH_MAX]; - - char dev[PATH_MAX]; - snprintf(dev, sizeof(dev), "/dev/rbd%s", dent->d_name); - - char name[RBD_MAX_IMAGE_NAME_SIZE]; - snprintf(fn, sizeof(fn), "%s/%s/name", devices_path, dent->d_name); - r = read_file(fn, name, sizeof(name)); - if (r < 0) { - cerr << "rbd: could not read name from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - - char pool[4096]; - snprintf(fn, sizeof(fn), "%s/%s/pool", devices_path, dent->d_name); - r = read_file(fn, pool, sizeof(pool)); - if (r < 0) { - cerr << "rbd: could not read name from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - - char snap[4096]; - snprintf(fn, sizeof(fn), "%s/%s/current_snap", devices_path, dent->d_name); - r = read_file(fn, snap, sizeof(snap)); - if (r < 0) { - cerr << "rbd: could not read name from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - - if (f) { - f->open_object_section(dent->d_name); - f->dump_string("pool", pool); - f->dump_string("name", name); - f->dump_string("snap", snap); - f->dump_string("device", dev); - f->close_section(); - } else { - tbl << dent->d_name << pool << name << snap << dev << TextTable::endrow; - } - have_output = true; - - } while ((dent = readdir(device_dir.get()))); - - if (f) { - f->close_section(); - f->flush(cout); - } else { - if (have_output) - cout << tbl; - } - - return 0; -} - -static int get_rbd_seq(dev_t devno, string &seq) -{ - // convert devno, which might be a partition major:minor pair, into - // a whole disk major:minor pair - dev_t wholediskno; - int r = blkid_devno_to_wholedisk(devno, NULL, 0, &wholediskno); - if (r) { - cerr << "rbd: could not compute wholediskno: " << r << std::endl; - // ignore the error: devno == wholediskno most of the time, and if - // it turns out it's not we will fail with -ENOENT later anyway - wholediskno = devno; - } - - const char *devices_path = "/sys/bus/rbd/devices"; - DIR *device_dir = opendir(devices_path); - if (!device_dir) { - r = -errno; - cerr << "rbd: could not open " << devices_path << ": " << cpp_strerror(-r) - << std::endl; - return r; - } - - struct dirent *dent; - dent = readdir(device_dir); - if (!dent) { - r = -errno; - cerr << "Error reading " << devices_path << ": " << cpp_strerror(-r) - << std::endl; - closedir(device_dir); + r = krbd_create_from_context(g_ceph_context, &krbd); + if (r < 0) return r; - } - - int match_minor = -1; - do { - char fn[strlen(devices_path) + strlen(dent->d_name) + strlen("//major") + 1]; - char buf[32]; - - if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) - continue; - - snprintf(fn, sizeof(fn), "%s/%s/major", devices_path, dent->d_name); - r = read_file(fn, buf, sizeof(buf)); - if (r < 0) { - cerr << "rbd: could not read major number from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - string err; - int cur_major = strict_strtol(buf, 10, &err); - if (!err.empty()) { - cerr << err << std::endl; - cerr << "rbd: could not parse major number read from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - if (cur_major != (int)major(wholediskno)) - continue; - - if (match_minor == -1) { - // matching minors in addition to majors is not necessary unless - // single-major scheme is turned on, but, if the kernel supports - // it, do it anyway (blkid stuff above ensures that we always have - // the correct minor to match with) - struct stat sbuf; - snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name); - match_minor = (stat(fn, &sbuf) == 0); - } - - if (match_minor == 1) { - snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name); - r = read_file(fn, buf, sizeof(buf)); - if (r < 0) { - cerr << "rbd: could not read minor number from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - int cur_minor = strict_strtol(buf, 10, &err); - if (!err.empty()) { - cerr << err << std::endl; - cerr << "rbd: could not parse minor number read from " << fn << ": " - << cpp_strerror(-r) << std::endl; - continue; - } - if (cur_minor != (int)minor(wholediskno)) - continue; - } else { - assert(match_minor == 0); - } - seq = string(dent->d_name); - closedir(device_dir); - return 0; - } while ((dent = readdir(device_dir))); + r = krbd_showmapped(krbd, f); - closedir(device_dir); - return -ENOENT; + krbd_destroy(krbd); + return r; } -static int do_kernel_rm(const char *dev) +static int do_kernel_unmap(const char *dev) { - struct stat sbuf; - if (stat(dev, &sbuf) || !S_ISBLK(sbuf.st_mode)) { - cerr << "rbd: " << dev << " is not a block device" << std::endl; - return -EINVAL; - } + struct krbd_ctx *krbd; + int r; - string seq_num; - int r = get_rbd_seq(sbuf.st_rdev, seq_num); - if (r == -ENOENT) { - cerr << "rbd: " << dev << " is not an rbd device" << std::endl; - return -EINVAL; - } + r = krbd_create_from_context(g_ceph_context, &krbd); if (r < 0) return r; - // let udevadm do its job *before* we try to unmap - if (udevadm_settle) { - int r = system("/sbin/udevadm settle"); - if (r) { - if (r < 0) - cerr << "rbd: error executing udevadm as shell command!" << std::endl; - else - cerr << "rbd: '/sbin/udevadm settle' failed! (" << r << ")" <