From 6afc947f61d2b1c1c55a276e61ae77b21ad6e25d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Piotr=20Da=C5=82ek?= Date: Wed, 7 Jun 2017 16:01:37 +0200 Subject: [PATCH] rbd: parallelize rbd ls -l MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When a cluster contains a large number of images, "rbd ls -l" takes a long time to finish. In my particular case, it took about 58s to process 3000 images. "rbd ls -l" opens each of image and that takes majority of time, so improve this by using aio_open() and aio_close() to do it asynchronously. This reduced total processing time down to around 15 seconds when using default 10 concurrently opened images. Signed-off-by: Piotr Dałek (cherry picked from commit 8f76fc861b0a628fa2269b04f77b7e31d4a7a006) --- src/tools/rbd/action/List.cc | 333 ++++++++++++++++++++++------------- 1 file changed, 206 insertions(+), 127 deletions(-) diff --git a/src/tools/rbd/action/List.cc b/src/tools/rbd/action/List.cc index 53bebd83d8c2..4c4babe99489 100644 --- a/src/tools/rbd/action/List.cc +++ b/src/tools/rbd/action/List.cc @@ -4,6 +4,7 @@ #include "tools/rbd/ArgumentTypes.h" #include "tools/rbd/Shell.h" #include "tools/rbd/Utils.h" +#include "include/Context.h" #include "include/stringify.h" #include "include/types.h" #include "common/errno.h" @@ -11,18 +12,158 @@ #include "common/TextTable.h" #include #include +#include "global/global_context.h" namespace rbd { + namespace action { namespace list { namespace at = argument_types; namespace po = boost::program_options; -int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag, - Formatter *f) { +enum WorkerState { + STATE_IDLE = 0, + STATE_OPENED, + STATE_DONE +} ; + +struct WorkerEntry { + librbd::Image img; + librbd::RBD::AioCompletion* completion; + WorkerState state; + string name; + + WorkerEntry() { + state = STATE_IDLE; + completion = nullptr; + } +}; + + +int list_process_image(librados::Rados* rados, WorkerEntry* w, bool lflag, Formatter *f, TextTable &tbl) +{ + int r = 0; + librbd::image_info_t info; + std::string pool, image, snap, parent; + + // handle second-nth trips through loop + r = w->img.parent_info(&pool, &image, &snap); + if (r < 0 && r != -ENOENT) + return r; + bool has_parent = false; + if (r != -ENOENT) { + parent = pool + "/" + image + "@" + snap; + has_parent = true; + } + + if (w->img.stat(info, sizeof(info)) < 0) { + return -EINVAL; + } + + uint8_t old_format; + w->img.old_format(&old_format); + + std::list lockers; + bool exclusive; + r = w->img.list_lockers(&lockers, &exclusive, NULL); + if (r < 0) + return r; + std::string lockstr; + if (!lockers.empty()) { + lockstr = (exclusive) ? "excl" : "shr"; + } + + if (f) { + f->open_object_section("image"); + f->dump_string("image", w->name); + f->dump_unsigned("size", info.size); + if (has_parent) { + f->open_object_section("parent"); + f->dump_string("pool", pool); + f->dump_string("image", image); + f->dump_string("snapshot", snap); + f->close_section(); + } + f->dump_int("format", old_format ? 1 : 2); + if (!lockers.empty()) + f->dump_string("lock_type", exclusive ? "exclusive" : "shared"); + f->close_section(); + } else { + tbl << w->name + << stringify(si_t(info.size)) + << parent + << ((old_format) ? '1' : '2') + << "" // protect doesn't apply to images + << lockstr + << TextTable::endrow; + } + + std::vector snaplist; + if (w->img.snap_list(snaplist) >= 0 && !snaplist.empty()) { + for (std::vector::iterator s = snaplist.begin(); + s != snaplist.end(); ++s) { + bool is_protected; + bool has_parent = false; + parent.clear(); + w->img.snap_set(s->name.c_str()); + r = w->img.snap_is_protected(s->name.c_str(), &is_protected); + if (r < 0) + return r; + if (w->img.parent_info(&pool, &image, &snap) >= 0) { + parent = pool + "/" + image + "@" + snap; + has_parent = true; + } + if (f) { + f->open_object_section("snapshot"); + f->dump_string("image", w->name); + f->dump_string("snapshot", s->name); + f->dump_unsigned("size", s->size); + if (has_parent) { + f->open_object_section("parent"); + f->dump_string("pool", pool); + f->dump_string("image", image); + f->dump_string("snapshot", snap); + f->close_section(); + } + f->dump_int("format", old_format ? 1 : 2); + f->dump_string("protected", is_protected ? "true" : "false"); + f->close_section(); + } else { + tbl << w->name + "@" + s->name + << stringify(si_t(s->size)) + << parent + << ((old_format) ? '1' : '2') + << (is_protected ? "yes" : "") + << "" // locks don't apply to snaps + << TextTable::endrow; + } + } + } + + return r < 0 ? r : 0; +} + +int do_list(std::string &pool_name, bool lflag, int threads, Formatter *f) { + std::vector workers; std::vector names; - int r = rbd.list(io_ctx, names); + librados::Rados rados; + librbd::RBD rbd; + librados::IoCtx ioctx; + + if (threads < 1) { + threads = 1; + } + if (threads > 32) { + threads = 32; + } + + int r = utils::init(pool_name, &rados, &ioctx); + if (r < 0) { + return r; + } + + r = rbd.list(ioctx, names); if (r < 0) return r; @@ -32,9 +173,9 @@ int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag, for (std::vector::const_iterator i = names.begin(); i != names.end(); ++i) { if (f) - f->dump_string("name", *i); + f->dump_string("name", *i); else - std::cout << *i << std::endl; + std::cout << *i << std::endl; } if (f) { f->close_section(); @@ -56,125 +197,65 @@ int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag, tbl.define_column("LOCK", TextTable::LEFT, TextTable::LEFT); } - std::string pool, image, snap, parent; - - for (std::vector::const_iterator i = names.begin(); - i != names.end(); ++i) { - librbd::image_info_t info; - librbd::Image im; - - r = rbd.open_read_only(io_ctx, im, i->c_str(), NULL); - // image might disappear between rbd.list() and rbd.open(); ignore - // that, warn about other possible errors (EPERM, say, for opening - // an old-format image, because you need execute permission for the - // class method) - if (r < 0) { - if (r != -ENOENT) { - std::cerr << "rbd: error opening " << *i << ": " << cpp_strerror(r) - << std::endl; - } - // in any event, continue to next image - continue; - } - - // handle second-nth trips through loop - parent.clear(); - r = im.parent_info(&pool, &image, &snap); - if (r < 0 && r != -ENOENT) - goto out; - bool has_parent = false; - if (r != -ENOENT) { - parent = pool + "/" + image + "@" + snap; - has_parent = true; - } - - if (im.stat(info, sizeof(info)) < 0) { - r = -EINVAL; - goto out; - } - - uint8_t old_format; - im.old_format(&old_format); - - std::list lockers; - bool exclusive; - r = im.list_lockers(&lockers, &exclusive, NULL); - if (r < 0) - goto out; - std::string lockstr; - if (!lockers.empty()) { - lockstr = (exclusive) ? "excl" : "shr"; - } + for (int left = 0; left < std::min(threads, (int)names.size()); left++) { + workers.push_back(new WorkerEntry()); + } - if (f) { - f->open_object_section("image"); - f->dump_string("image", *i); - f->dump_unsigned("size", info.size); - if (has_parent) { - f->open_object_section("parent"); - f->dump_string("pool", pool); - f->dump_string("image", image); - f->dump_string("snapshot", snap); - f->close_section(); + auto i = names.begin(); + while (true) { + size_t workers_idle = 0; + for (auto comp : workers) { + switch (comp->state) { + case STATE_DONE: + comp->completion->wait_for_complete(); + comp->state = STATE_IDLE; + comp->completion->release(); + comp->completion = nullptr; + // we want it to fall through in this case + case STATE_IDLE: + if (i == names.end()) { + workers_idle++; + continue; + } + comp->name = *i; + comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr); + r = rbd.aio_open_read_only(ioctx, comp->img, i->c_str(), NULL, comp->completion); + i++; + comp->state = STATE_OPENED; + break; + case STATE_OPENED: + comp->completion->wait_for_complete(); + // image might disappear between rbd.list() and rbd.open(); ignore + // that, warn about other possible errors (EPERM, say, for opening + // an old-format image, because you need execute permission for the + // class method) + r = comp->completion->get_return_value(); + comp->completion->release(); + if (r < 0) { + if (r != -ENOENT) { + std::cerr << "rbd: error opening " << *i << ": " << cpp_strerror(r) + << std::endl; + } + // in any event, continue to next image + comp->state = STATE_IDLE; + continue; + } + r = list_process_image(&rados, comp, lflag, f, tbl); + if (r < 0) { + std::cerr << "rbd: error processing image " << comp->name << ": " << cpp_strerror(r) + << std::endl; + } + comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr); + r = comp->img.aio_close(comp->completion); + comp->state = STATE_DONE; + break; } - f->dump_int("format", old_format ? 1 : 2); - if (!lockers.empty()) - f->dump_string("lock_type", exclusive ? "exclusive" : "shared"); - f->close_section(); - } else { - tbl << *i - << stringify(si_t(info.size)) - << parent - << ((old_format) ? '1' : '2') - << "" // protect doesn't apply to images - << lockstr - << TextTable::endrow; } - - std::vector snaplist; - if (im.snap_list(snaplist) >= 0 && !snaplist.empty()) { - for (std::vector::iterator s = snaplist.begin(); - s != snaplist.end(); ++s) { - bool is_protected; - bool has_parent = false; - parent.clear(); - im.snap_set(s->name.c_str()); - r = im.snap_is_protected(s->name.c_str(), &is_protected); - if (r < 0) - goto out; - if (im.parent_info(&pool, &image, &snap) >= 0) { - parent = pool + "/" + image + "@" + snap; - has_parent = true; - } - if (f) { - f->open_object_section("snapshot"); - f->dump_string("image", *i); - f->dump_string("snapshot", s->name); - f->dump_unsigned("size", s->size); - if (has_parent) { - f->open_object_section("parent"); - f->dump_string("pool", pool); - f->dump_string("image", image); - f->dump_string("snapshot", snap); - f->close_section(); - } - f->dump_int("format", old_format ? 1 : 2); - f->dump_string("protected", is_protected ? "true" : "false"); - f->close_section(); - } else { - tbl << *i + "@" + s->name - << stringify(si_t(s->size)) - << parent - << ((old_format) ? '1' : '2') - << (is_protected ? "yes" : "") - << "" // locks don't apply to snaps - << TextTable::endrow; - } - } + if (workers_idle == workers.size()) { + break; } } -out: if (f) { f->close_section(); f->flush(std::cout); @@ -182,6 +263,12 @@ out: std::cout << tbl; } + rados.shutdown(); + + for (auto comp : workers) { + delete comp; + } + return r < 0 ? r : 0; } @@ -203,15 +290,7 @@ int execute(const po::variables_map &vm) { return r; } - librados::Rados rados; - librados::IoCtx io_ctx; - r = utils::init(pool_name, &rados, &io_ctx); - if (r < 0) { - return r; - } - - librbd::RBD rbd; - r = do_list(rbd, io_ctx, vm["long"].as(), formatter.get()); + r = do_list(pool_name, vm["long"].as(), g_conf->rbd_concurrent_management_ops, formatter.get()); if (r < 0) { std::cerr << "rbd: list: " << cpp_strerror(r) << std::endl; return r; -- 2.47.3