From: Dongsheng Yang Date: Thu, 28 Jul 2016 11:06:44 +0000 (-0400) Subject: rbd: introduce --import-format to import command X-Git-Tag: v12.0.1~342^2~14 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5e034600a236136331a5d65623103ee21bbaee96;p=ceph.git rbd: introduce --import-format to import command Signed-off-by: Dongsheng Yang --- diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 215ca9acb62a..d848214d58e6 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -227,7 +227,7 @@ Commands :command:`export` (*image-spec* | *snap-spec*) [*dest-path*] Exports image to dest path (use - for stdout). -:command:`import` [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*] +:command:`import` [--import-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*] Creates a new image and imports its data from path (use - for stdin). The import operation will try to create sparse rbd images if possible. For import from stdin, the sparsification unit is @@ -236,6 +236,9 @@ Commands The --stripe-unit and --stripe-count arguments are optional, but must be used together. + The --import-format accepts '1' or '2' currently. Format 2 allow us to import not only the content + of image, but also the snapshots and other priorities, such as image_order, features. + :command:`export-diff` [--from-snap *snap-name*] [--whole-object] (*image-spec* | *snap-spec*) *dest-path* Exports an incremental diff for an image to dest path (use - for stdout). If an initial snapshot is specified, only changes since that snapshot are included; otherwise, diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 0158027d49dd..820b7ea74d1c 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -585,7 +585,8 @@ [--journal-splay-width ] [--journal-object-size ] [--journal-pool ] [--no-progress] - [--pool ] [--image ] + [--import-format ] [--pool ] + [--image ] Import image from file. @@ -616,6 +617,7 @@ --journal-object-size arg size of journal objects --journal-pool arg pool for journal objects --no-progress disable progress output + --import-format arg format of the file to be imported -p [ --pool ] arg pool name (deprecated) --image arg image name (deprecated) diff --git a/src/tools/rbd/Utils.h b/src/tools/rbd/Utils.h index b13e555a8c16..54250f170213 100644 --- a/src/tools/rbd/Utils.h +++ b/src/tools/rbd/Utils.h @@ -16,6 +16,10 @@ namespace utils { static const std::string RBD_DIFF_BANNER ("rbd diff v1\n"); +static const std::string RBD_IMAGE_BANNER_V2 ("rbd image v2\n"); +static const std::string RBD_IMAGE_DIFFS_BANNER_V2 ("rbd image diffss v2\n"); +static const std::string RBD_DIFF_BANNER_V2 ("rbd diff v2\n"); + #define RBD_DIFF_FROM_SNAP 'f' #define RBD_DIFF_TO_SNAP 't' #define RBD_DIFF_IMAGE_SIZE 's' diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc index 9d039f1cec1b..b72689d454d0 100644 --- a/src/tools/rbd/action/Import.cc +++ b/src/tools/rbd/action/Import.cc @@ -21,7 +21,7 @@ namespace rbd { namespace action { int do_import_diff_fd(librbd::Image &image, int fd, - bool no_progress) + bool no_progress, int format) { int r; struct stat stat_buf; @@ -29,7 +29,8 @@ int do_import_diff_fd(librbd::Image &image, int fd, uint64_t size = 0; uint64_t off = 0; string from, to; - char buf[utils::RBD_DIFF_BANNER.size() + 1]; + string banner = (format == 1 ? utils::RBD_DIFF_BANNER:utils::RBD_DIFF_BANNER_V2); + char buf[banner.size() + 1]; bool from_stdin = (fd == 0); if (!from_stdin) { @@ -39,13 +40,13 @@ int do_import_diff_fd(librbd::Image &image, int fd, size = (uint64_t)stat_buf.st_size; } - r = safe_read_exact(fd, buf, utils::RBD_DIFF_BANNER.size()); + r = safe_read_exact(fd, buf, banner.size()); if (r < 0) goto done; - buf[utils::RBD_DIFF_BANNER.size()] = '\0'; - if (strcmp(buf, utils::RBD_DIFF_BANNER.c_str())) { + buf[banner.size()] = '\0'; + if (strcmp(buf, banner.c_str())) { std::cerr << "invalid banner '" << buf << "', expected '" - << utils::RBD_DIFF_BANNER << "'" << std::endl; + << banner << "'" << std::endl; r = -EINVAL; goto done; } @@ -178,7 +179,7 @@ int do_import_diff(librbd::Image &image, const char *path, return r; } } - r = do_import_diff_fd(image, fd, no_progress); + r = do_import_diff_fd(image, fd, no_progress, 1); if (fd != 0) close(fd); @@ -290,8 +291,15 @@ private: static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx, const char *imgname, const char *path, - librbd::ImageOptions& opts, bool no_progress) + librbd::ImageOptions& opts, bool no_progress, + int import_format) { + // check current supported formats. + if (import_format != 1 && import_format != 2) { + std::cerr << "rbd: wrong file format to import" << std::endl; + return -EINVAL; + } + int fd, r; struct stat stat_buf; utils::ProgressContext pc("Importing image", no_progress); @@ -312,6 +320,8 @@ static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx, size_t blklen = 0; // amount accumulated from reads to fill blk librbd::Image image; uint64_t size = 0; + uint64_t snap_num; + char image_buf[utils::RBD_IMAGE_BANNER_V2.size() + 1]; boost::scoped_ptr throttle; bool from_stdin = !strcmp(path, "-"); @@ -356,6 +366,28 @@ static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx, posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); #endif } + + if (import_format == 2) { + if (from_stdin || size < utils::RBD_IMAGE_BANNER_V2.size()) { + r = -EINVAL; + goto done; + } + r = safe_read_exact(fd, image_buf, utils::RBD_IMAGE_BANNER_V2.size()); + if (r < 0) + goto done; + + image_buf[utils::RBD_IMAGE_BANNER_V2.size()] = '\0'; + if (strcmp(image_buf, utils::RBD_IMAGE_BANNER_V2.c_str())) { + // Old format + r = -EINVAL; + goto done; + } + + r = safe_read_exact(fd, &size, 8); + if (r < 0) { + goto done; + } + } r = rbd.create4(io_ctx, imgname, size, opts); if (r < 0) { @@ -369,59 +401,90 @@ static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx, goto err; } - // loop body handles 0 return, as we may have a block to flush - while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) { - if (throttle->pending_error()) { - break; - } + if (import_format == 2) { + char snap_buf[utils::RBD_IMAGE_SNAPS_BANNER_V2.size() + 1]; - blklen += readlen; - // if read was short, try again to fill the block before writing - if (readlen && ((size_t)readlen < reqlen)) { - reqlen -= readlen; - continue; - } - if (!from_stdin) - pc.update_progress(image_pos, size); - - bufferlist bl(blklen); - bl.append(p, blklen); - // resize output image by binary expansion as we go for stdin - if (from_stdin && (image_pos + (size_t)blklen) > size) { - size *= 2; - r = image.resize(size); + r = safe_read_exact(fd, snap_buf, utils::RBD_IMAGE_SNAPS_BANNER_V2.size()); + if (r < 0) + goto done; + snap_buf[utils::RBD_IMAGE_SNAPS_BANNER_V2.size()] = '\0'; + if (strcmp(snap_buf, utils::RBD_IMAGE_SNAPS_BANNER_V2.c_str())) { + cerr << "Incorrect RBD_IMAGE_SNAPS_BANNER." << std::endl; + return -EINVAL; + } else { + r = safe_read_exact(fd, &snap_num, 8); if (r < 0) { - std::cerr << "rbd: can't resize image during import" << std::endl; goto close; } } - // write as much as we got; perhaps less than imgblklen - // but skip writing zeros to create sparse images - if (!bl.is_zero()) { - C_Import *ctx = new C_Import(*throttle, image, bl, image_pos); - ctx->send(); + for (size_t i = 0; i < snap_num; i++) { + r = do_import_diff_fd(image, fd, true, 2); + if (r < 0) { + pc.fail(); + std::cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl; + return r; + } + pc.update_progress(i + 1, snap_num); } + } else { + reqlen = min(reqlen, size); + // loop body handles 0 return, as we may have a block to flush + while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) { + if (throttle->pending_error()) { + break; + } - // done with whole block, whether written or not - image_pos += blklen; - // if read had returned 0, we're at EOF and should quit - if (readlen == 0) - break; - blklen = 0; - reqlen = imgblklen; - } - r = throttle->wait_for_ret(); - if (r < 0) { - goto close; - } + blklen += readlen; + // if read was short, try again to fill the block before writing + if (readlen && ((size_t)readlen < reqlen)) { + reqlen -= readlen; + continue; + } + if (!from_stdin) + pc.update_progress(image_pos, size); + + bufferlist bl(blklen); + bl.append(p, blklen); + // resize output image by binary expansion as we go for stdin + if (from_stdin && (image_pos + (size_t)blklen) > size) { + size *= 2; + r = image.resize(size); + if (r < 0) { + std::cerr << "rbd: can't resize image during import" << std::endl; + goto close; + } + } - if (from_stdin) { - r = image.resize(image_pos); + // write as much as we got; perhaps less than imgblklen + // but skip writing zeros to create sparse images + if (!bl.is_zero()) { + C_Import *ctx = new C_Import(*throttle, image, bl, image_pos); + ctx->send(); + } + + // done with whole block, whether written or not + image_pos += blklen; + if (!from_stdin && image_pos >= size) + break; + // if read had returned 0, we're at EOF and should quit + if (readlen == 0) + break; + blklen = 0; + reqlen = imgblklen; + } + r = throttle->wait_for_ret(); if (r < 0) { - std::cerr << "rbd: final image resize failed" << std::endl; goto close; } + + if (from_stdin) { + r = image.resize(image_pos); + if (r < 0) { + std::cerr << "rbd: final image resize failed" << std::endl; + goto close; + } + } } close: @@ -434,7 +497,8 @@ done: pc.fail(); else pc.finish(); - close(fd); + if (!from_stdin) + close(fd); done2: delete[] p; return r; @@ -447,6 +511,8 @@ void get_arguments(po::options_description *positional, at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST); at::add_create_image_options(options, true); at::add_no_progress_option(options); + options->add_options() + ("import-format", po::value(), "format of the file to be imported"); // TODO legacy rbd allowed import to accept both 'image'/'dest' and // 'pool'/'dest-pool' @@ -509,9 +575,13 @@ int execute(const po::variables_map &vm) { return r; } + int format = 1; + if (vm.count("import-format")) + format = vm["import-format"].as(); + librbd::RBD rbd; r = do_import(rbd, io_ctx, image_name.c_str(), path.c_str(), - opts, vm[at::NO_PROGRESS].as()); + opts, vm[at::NO_PROGRESS].as(), format); if (r < 0) { std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl; return r;