rbd: add --estimated-size option for import from stdin

One issue with importing from stdin is that it's not easy to track
progress. The only feasible option is to process messages on the highest
log level looking for lines like

librbd::io::ImageRequestWQ: 0x56342ecc7a50 aio_write: ... off=1187840, len=864256 ...

but when it comes to large images it takes a lot of effort.
This commit introduces --estimated-size option, that makes it possible
to print out progress in percents via the standard mechanism. Obviously,
it requires the knowledge of the amount of provided data in advance and
in case of an error nonsensical percents might be printed, but I don't
think it's that big of a deal.

Also use `estimated size` as the base image size, making resizing not
necessary in cases where we know the exact amount of data provided from
stdin.

Co-authored-by: Ilya Dryomov <idryomov@gmail.com>
Signed-off-by: Kirill Nazarov <KiNazarov@croc.ru>
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
index 6956b9e334bc1c4478d9cfc7b8bd0c13d74667f3..55fa96e33285f953223fb89dd5af1bd4078b1d15 100644 (file)
--- a/doc/man/8/rbd.rst
+++ b/doc/man/8/rbd.rst
@@ -392,7 +392,7 @@ Commands
 :command:`image-meta set` *image-spec* *key* *value*
   Set metadata key with the value. They will displayed in `image-meta list`.
 
-:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--estimated-size *size-in-M/G/T*]... [--image-shared] *src-path* [*image-spec*]
   Create a new image and import its data from path (use - for
   stdin).  The import operation will try to create sparse rbd images 
   if possible.  For import from stdin, the sparsification unit is

diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index b94bd9bd7eeb1b90990ea40b404979f8416364f9..b4149618bb659e4e849eb84a16404c32553c1116 100644 (file)
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -1258,7 +1258,8 @@
                     [--journal-object-size <journal-object-size>] 
                     [--journal-pool <journal-pool>] 
                     [--sparse-size <sparse-size>] [--no-progress] 
-                    [--export-format <export-format>] [--pool <pool>] 
+                    [--export-format <export-format>] 
+                    [--estimated-size <estimated-size>] [--pool <pool>] 
                     [--image <image>] 
                     <path-name> <dest-image-spec> 
   
@@ -1290,6 +1291,8 @@
     --sparse-size arg         sparse size in B/K/M [default: 4K]
     --no-progress             disable progress output
     --export-format arg       format of image file
+    --estimated-size arg      estimated image size (valid only for raw import
+                              from stdin, in M/G/T) [default: M]
   
   Image Features:
     (*) supports enabling/disabling on existing images

diff --git a/src/tools/rbd/ArgumentTypes.cc b/src/tools/rbd/ArgumentTypes.cc
index b479f96158845df66af434322a9e3d32e2864f6a..7d60a95ca4be49e44ad383304801bd3adfab3756 100644 (file)
--- a/src/tools/rbd/ArgumentTypes.cc
+++ b/src/tools/rbd/ArgumentTypes.cc
@@ -279,6 +279,12 @@ void add_size_option(boost::program_options::options_description *opt) {
      "image size (in M/G/T) [default: M]");
 }
 
+void add_estimated_size_option(boost::program_options::options_description *opt) {
+  opt->add_options()
+    (IMAGE_ESTIMATED_SIZE.c_str(), po::value<ImageSize>(),
+     "estimated image size (valid only for raw import from stdin, in M/G/T) [default: M]");
+}
+
 void add_sparse_size_option(boost::program_options::options_description *opt) {
   opt->add_options()
     (IMAGE_SPARSE_SIZE.c_str(), po::value<ImageObjectSize>(),

diff --git a/src/tools/rbd/ArgumentTypes.h b/src/tools/rbd/ArgumentTypes.h
index cc7c4813636930a7f9e89b844cca2a1da6bc2488..411d4cb78e6866a5b3e6645befc80af89669be6a 100644 (file)
--- a/src/tools/rbd/ArgumentTypes.h
+++ b/src/tools/rbd/ArgumentTypes.h
@@ -67,6 +67,7 @@ static const std::string IMAGE_OBJECT_SIZE("object-size");
 static const std::string IMAGE_FEATURES("image-feature");
 static const std::string IMAGE_SHARED("image-shared");
 static const std::string IMAGE_SIZE("size");
+static const std::string IMAGE_ESTIMATED_SIZE("estimated-size");
 static const std::string IMAGE_STRIPE_UNIT("stripe-unit");
 static const std::string IMAGE_STRIPE_COUNT("stripe-count");
 static const std::string IMAGE_DATA_POOL("data-pool");
@@ -186,6 +187,8 @@ void add_create_journal_options(
 
 void add_size_option(boost::program_options::options_description *opt);
 
+void add_estimated_size_option(boost::program_options::options_description *opt);
+
 void add_sparse_size_option(boost::program_options::options_description *opt);
 
 void add_path_options(boost::program_options::options_description *pos,

diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc
index f6b3c4c3ccd8f8d5ae2d378376b0098997dcf064..0e1d3eaf7ac850bd8ab0c233d53f646815c25af4 100644 (file)
--- a/src/tools/rbd/action/Import.cc
+++ b/src/tools/rbd/action/Import.cc
@@ -725,7 +725,7 @@ static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
 
 static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
                         size_t imgblklen, utils::ProgressContext &pc,
-                       size_t sparse_size)
+                        size_t sparse_size, size_t estimated_size)
 {
   int r = 0;
   size_t reqlen = imgblklen;    // amount requested from read
@@ -758,6 +758,8 @@ static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
     }
     if (!from_stdin)
       pc.update_progress(image_pos, size);
+    else if (estimated_size != 0)
+      pc.update_progress(image_pos, estimated_size);
 
     bufferptr blkptr(p, blklen); 
     // resize output image by binary expansion as we go for stdin
@@ -823,7 +825,8 @@ out:
 static int do_import(librados::Rados &rados, librbd::RBD &rbd,
                     librados::IoCtx& io_ctx, const char *imgname,
                     const char *path, librbd::ImageOptions& opts,
-                    bool no_progress, int import_format, size_t sparse_size)
+                    bool no_progress, int import_format, size_t sparse_size,
+                    size_t estimated_size)
 {
   int fd, r;
   struct stat stat_buf;
@@ -845,7 +848,11 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   bool from_stdin = !strcmp(path, "-");
   if (from_stdin) {
     fd = STDIN_FILENO;
-    size = 1ULL << order;
+    if (estimated_size == 0) {
+      size = 1ULL << order;
+    } else {
+      size = estimated_size;
+    }
   } else {
     if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) {
       r = -errno;
@@ -908,7 +915,8 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd,
   }
 
   if (import_format == 1) {
-    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
+    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size,
+                     estimated_size);
   } else {
     r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
   }
@@ -942,6 +950,7 @@ void get_arguments(po::options_description *positional,
   at::add_sparse_size_option(options);
   at::add_no_progress_option(options);
   at::add_export_format_option(options);
+  at::add_estimated_size_option(options);
 
   // TODO legacy rbd allowed import to accept both 'image'/'dest' and
   //      'pool'/'dest-pool'
@@ -1017,9 +1026,25 @@ int execute(const po::variables_map &vm,
   if (vm.count("export-format"))
     format = vm["export-format"].as<uint64_t>();
 
+  size_t estimated_size = 0;
+  if (vm.count(at::IMAGE_ESTIMATED_SIZE)) {
+    if (path != "-") {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for import from stdin" << std::endl;
+      return -EINVAL;
+    }
+    if (format != 1) {
+      std::cerr << "rbd: --estimated-size can be specified "
+                << "only for raw import (--export-format 1)" << std::endl;
+      return -EINVAL;
+    }
+    estimated_size = vm[at::IMAGE_ESTIMATED_SIZE].as<size_t>();
+  }
+
   librbd::RBD rbd;
   r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
-                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
+                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size,
+                estimated_size);
   if (r < 0) {
     std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
     return r;