From b4fbabd6fca55f081226f8f768e6fbe0037d215c Mon Sep 17 00:00:00 2001 From: Kirill Nazarov Date: Sun, 26 Jan 2025 22:08:24 +0300 Subject: [PATCH] rbd: add --estimated-size option for import from stdin One issue with importing from stdin is that it's not easy to track progress. The only feasible option is to process messages on the highest log level looking for lines like librbd::io::ImageRequestWQ: 0x56342ecc7a50 aio_write: ... off=1187840, len=864256 ... but when it comes to large images it takes a lot of effort. This commit introduces --estimated-size option, that makes it possible to print out progress in percents via the standard mechanism. Obviously, it requires the knowledge of the amount of provided data in advance and in case of an error nonsensical percents might be printed, but I don't think it's that big of a deal. Also use `estimated size` as the base image size, making resizing not necessary in cases where we know the exact amount of data provided from stdin. Co-authored-by: Ilya Dryomov Signed-off-by: Kirill Nazarov Signed-off-by: Ilya Dryomov --- doc/man/8/rbd.rst | 2 +- src/test/cli/rbd/help.t | 5 ++++- src/tools/rbd/ArgumentTypes.cc | 6 ++++++ src/tools/rbd/ArgumentTypes.h | 3 +++ src/tools/rbd/action/Import.cc | 35 +++++++++++++++++++++++++++++----- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 6956b9e334bc..55fa96e33285 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -392,7 +392,7 @@ Commands :command:`image-meta set` *image-spec* *key* *value* Set metadata key with the value. They will displayed in `image-meta list`. -:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*] +:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--estimated-size *size-in-M/G/T*]... [--image-shared] *src-path* [*image-spec*] Create a new image and import its data from path (use - for stdin). The import operation will try to create sparse rbd images if possible. For import from stdin, the sparsification unit is diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index b94bd9bd7eeb..b4149618bb65 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -1258,7 +1258,8 @@ [--journal-object-size ] [--journal-pool ] [--sparse-size ] [--no-progress] - [--export-format ] [--pool ] + [--export-format ] + [--estimated-size ] [--pool ] [--image ] @@ -1290,6 +1291,8 @@ --sparse-size arg sparse size in B/K/M [default: 4K] --no-progress disable progress output --export-format arg format of image file + --estimated-size arg estimated image size (valid only for raw import + from stdin, in M/G/T) [default: M] Image Features: (*) supports enabling/disabling on existing images diff --git a/src/tools/rbd/ArgumentTypes.cc b/src/tools/rbd/ArgumentTypes.cc index b479f9615884..7d60a95ca4be 100644 --- a/src/tools/rbd/ArgumentTypes.cc +++ b/src/tools/rbd/ArgumentTypes.cc @@ -279,6 +279,12 @@ void add_size_option(boost::program_options::options_description *opt) { "image size (in M/G/T) [default: M]"); } +void add_estimated_size_option(boost::program_options::options_description *opt) { + opt->add_options() + (IMAGE_ESTIMATED_SIZE.c_str(), po::value(), + "estimated image size (valid only for raw import from stdin, in M/G/T) [default: M]"); +} + void add_sparse_size_option(boost::program_options::options_description *opt) { opt->add_options() (IMAGE_SPARSE_SIZE.c_str(), po::value(), diff --git a/src/tools/rbd/ArgumentTypes.h b/src/tools/rbd/ArgumentTypes.h index cc7c48136369..411d4cb78e68 100644 --- a/src/tools/rbd/ArgumentTypes.h +++ b/src/tools/rbd/ArgumentTypes.h @@ -67,6 +67,7 @@ static const std::string IMAGE_OBJECT_SIZE("object-size"); static const std::string IMAGE_FEATURES("image-feature"); static const std::string IMAGE_SHARED("image-shared"); static const std::string IMAGE_SIZE("size"); +static const std::string IMAGE_ESTIMATED_SIZE("estimated-size"); static const std::string IMAGE_STRIPE_UNIT("stripe-unit"); static const std::string IMAGE_STRIPE_COUNT("stripe-count"); static const std::string IMAGE_DATA_POOL("data-pool"); @@ -186,6 +187,8 @@ void add_create_journal_options( void add_size_option(boost::program_options::options_description *opt); +void add_estimated_size_option(boost::program_options::options_description *opt); + void add_sparse_size_option(boost::program_options::options_description *opt); void add_path_options(boost::program_options::options_description *pos, diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc index f6b3c4c3ccd8..0e1d3eaf7ac8 100644 --- a/src/tools/rbd/action/Import.cc +++ b/src/tools/rbd/action/Import.cc @@ -725,7 +725,7 @@ static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image, static int do_import_v1(int fd, librbd::Image &image, uint64_t size, size_t imgblklen, utils::ProgressContext &pc, - size_t sparse_size) + size_t sparse_size, size_t estimated_size) { int r = 0; size_t reqlen = imgblklen; // amount requested from read @@ -758,6 +758,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size, } if (!from_stdin) pc.update_progress(image_pos, size); + else if (estimated_size != 0) + pc.update_progress(image_pos, estimated_size); bufferptr blkptr(p, blklen); // resize output image by binary expansion as we go for stdin @@ -823,7 +825,8 @@ out: static int do_import(librados::Rados &rados, librbd::RBD &rbd, librados::IoCtx& io_ctx, const char *imgname, const char *path, librbd::ImageOptions& opts, - bool no_progress, int import_format, size_t sparse_size) + bool no_progress, int import_format, size_t sparse_size, + size_t estimated_size) { int fd, r; struct stat stat_buf; @@ -845,7 +848,11 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd, bool from_stdin = !strcmp(path, "-"); if (from_stdin) { fd = STDIN_FILENO; - size = 1ULL << order; + if (estimated_size == 0) { + size = 1ULL << order; + } else { + size = estimated_size; + } } else { if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) { r = -errno; @@ -908,7 +915,8 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd, } if (import_format == 1) { - r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size); + r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size, + estimated_size); } else { r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size); } @@ -942,6 +950,7 @@ void get_arguments(po::options_description *positional, at::add_sparse_size_option(options); at::add_no_progress_option(options); at::add_export_format_option(options); + at::add_estimated_size_option(options); // TODO legacy rbd allowed import to accept both 'image'/'dest' and // 'pool'/'dest-pool' @@ -1017,9 +1026,25 @@ int execute(const po::variables_map &vm, if (vm.count("export-format")) format = vm["export-format"].as(); + size_t estimated_size = 0; + if (vm.count(at::IMAGE_ESTIMATED_SIZE)) { + if (path != "-") { + std::cerr << "rbd: --estimated-size can be specified " + << "only for import from stdin" << std::endl; + return -EINVAL; + } + if (format != 1) { + std::cerr << "rbd: --estimated-size can be specified " + << "only for raw import (--export-format 1)" << std::endl; + return -EINVAL; + } + estimated_size = vm[at::IMAGE_ESTIMATED_SIZE].as(); + } + librbd::RBD rbd; r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(), - opts, vm[at::NO_PROGRESS].as(), format, sparse_size); + opts, vm[at::NO_PROGRESS].as(), format, sparse_size, + estimated_size); if (r < 0) { std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl; return r; -- 2.47.3