]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
librbd: add encryption api 38712/head
authorOr Ozeri <oro@il.ibm.com>
Sun, 10 Jan 2021 16:46:28 +0000 (18:46 +0200)
committerOr Ozeri <oro@il.ibm.com>
Mon, 11 Jan 2021 11:55:38 +0000 (13:55 +0200)
This commit exposes librbd encryption api

Signed-off-by: Or Ozeri <oro@il.ibm.com>
38 files changed:
src/include/config-h.in.cmake
src/include/rbd/librbd.h
src/include/rbd/librbd.hpp
src/librbd/CMakeLists.txt
src/librbd/ImageCtx.cc
src/librbd/ImageCtx.h
src/librbd/api/Image.cc
src/librbd/api/Image.h
src/librbd/api/Utils.cc [new file with mode: 0644]
src/librbd/api/Utils.h [new file with mode: 0644]
src/librbd/crypto/CryptoImageDispatch.h
src/librbd/crypto/CryptoObjectDispatch.cc
src/librbd/crypto/CryptoObjectDispatch.h
src/librbd/crypto/EncryptionFormat.h [new file with mode: 0644]
src/librbd/crypto/FormatRequest.cc [new file with mode: 0644]
src/librbd/crypto/FormatRequest.h [new file with mode: 0644]
src/librbd/crypto/LoadRequest.cc [new file with mode: 0644]
src/librbd/crypto/LoadRequest.h [new file with mode: 0644]
src/librbd/crypto/Types.h
src/librbd/crypto/luks/EncryptionFormat.cc [new file with mode: 0644]
src/librbd/crypto/luks/EncryptionFormat.h [new file with mode: 0644]
src/librbd/crypto/luks/FormatRequest.cc
src/librbd/crypto/luks/FormatRequest.h
src/librbd/crypto/luks/Header.cc
src/librbd/crypto/luks/Header.h
src/librbd/crypto/luks/LoadRequest.cc
src/librbd/crypto/luks/LoadRequest.h
src/librbd/internal.cc
src/librbd/io/ImageRequest.cc
src/librbd/librbd.cc
src/test/librbd/CMakeLists.txt
src/test/librbd/crypto/luks/test_mock_FormatRequest.cc
src/test/librbd/crypto/luks/test_mock_LoadRequest.cc
src/test/librbd/crypto/test_mock_FormatRequest.cc [new file with mode: 0644]
src/test/librbd/crypto/test_mock_LoadRequest.cc [new file with mode: 0644]
src/test/librbd/mock/MockImageCtx.h
src/test/librbd/mock/crypto/MockEncryptionFormat.h [new file with mode: 0644]
src/test/librbd/test_librbd.cc

index ca5971cbe910fc64014f8aceb22b38885f37b64a..9a4e1a6479f5cdbc6c436c8acf9c873b91d9483c 100644 (file)
 /* Define if libcryptsetup version < 2.0.5 */
 #cmakedefine LIBCRYPTSETUP_LEGACY_DATA_ALIGNMENT
 
+/* Define if libcryptsetup can be used (linux only) */
+#cmakedefine HAVE_LIBCRYPTSETUP
+
 /* Shared library extension, such as .so, .dll or .dylib */
 #cmakedefine CMAKE_SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@"
 
index a0eb57eef3a72148ebb02a5d42c80b4e3c4be88b..c96ed18656002c0657dec2c5d8af5dcfce278435 100644 (file)
@@ -372,6 +372,30 @@ enum {
   RBD_WRITE_ZEROES_FLAG_THICK_PROVISION = (1U<<0), /* fully allocated zeroed extent */
 };
 
+typedef enum {
+    RBD_ENCRYPTION_FORMAT_LUKS1 = 0,
+    RBD_ENCRYPTION_FORMAT_LUKS2 = 1
+} rbd_encryption_format_t;
+
+typedef enum {
+    RBD_ENCRYPTION_ALGORITHM_AES128 = 0,
+    RBD_ENCRYPTION_ALGORITHM_AES256 = 1
+} rbd_encryption_algorithm_t;
+
+typedef void *rbd_encryption_options_t;
+
+typedef struct {
+    rbd_encryption_algorithm_t alg;
+    const char* passphrase;
+    size_t passphrase_size;
+} rbd_encryption_luks1_format_options_t;
+
+typedef struct {
+    rbd_encryption_algorithm_t alg;
+    const char* passphrase;
+    size_t passphrase_size;
+} rbd_encryption_luks2_format_options_t;
+
 CEPH_RBD_API void rbd_image_options_create(rbd_image_options_t* opts);
 CEPH_RBD_API void rbd_image_options_destroy(rbd_image_options_t opts);
 CEPH_RBD_API int rbd_image_options_set_string(rbd_image_options_t opts,
@@ -791,6 +815,16 @@ CEPH_RBD_API int rbd_deep_copy_with_progress(rbd_image_t image,
                                              librbd_progress_fn_t cb,
                                              void *cbdata);
 
+/* encryption */
+CEPH_RBD_API int rbd_encryption_format(rbd_image_t image,
+                                       rbd_encryption_format_t format,
+                                       rbd_encryption_options_t opts,
+                                       size_t opts_size);
+CEPH_RBD_API int rbd_encryption_load(rbd_image_t image,
+                                     rbd_encryption_format_t format,
+                                     rbd_encryption_options_t opts,
+                                     size_t opts_size);
+
 /* snapshots */
 CEPH_RBD_API int rbd_snap_list(rbd_image_t image, rbd_snap_info_t *snaps,
                                int *max_snaps);
index 71e955b25249415f1f019a8830489da5ff0c7faa..f1ddc2965e5451143fd09d8097481af519413ed3 100644 (file)
@@ -214,6 +214,20 @@ namespace librbd {
     config_source_t source;
   } config_option_t;
 
+  typedef rbd_encryption_format_t encryption_format_t;
+  typedef rbd_encryption_algorithm_t encryption_algorithm_t;
+  typedef rbd_encryption_options_t encryption_options_t;
+
+  typedef struct {
+    encryption_algorithm_t alg;
+    std::string passphrase;
+  } encryption_luks1_format_options_t;
+
+  typedef struct {
+    encryption_algorithm_t alg;
+    std::string passphrase;
+  } encryption_luks2_format_options_t;
+
 class CEPH_RBD_API RBD
 {
 public:
@@ -576,6 +590,12 @@ public:
   int deep_copy_with_progress(IoCtx& dest_io_ctx, const char *destname,
                               ImageOptions& opts, ProgressContext &prog_ctx);
 
+  /* encryption */
+  int encryption_format(encryption_format_t format, encryption_options_t opts,
+                        size_t opts_size);
+  int encryption_load(encryption_format_t format, encryption_options_t opts,
+                      size_t opts_size);
+
   /* striping */
   uint64_t get_stripe_unit() const;
   uint64_t get_stripe_count() const;
index 20bfd5427967b18d9876345b495ed3532ee0e159..aa99dc9bd55470587900d92ccedc85a9eef9ec18 100644 (file)
@@ -54,6 +54,7 @@ set(librbd_internal_srcs
   api/PoolMetadata.cc
   api/Snapshot.cc
   api/Trash.cc
+  api/Utils.cc
   asio/ContextWQ.cc
   cache/ImageWriteback.cc
   cache/ObjectCacherObjectDispatch.cc
@@ -63,6 +64,8 @@ set(librbd_internal_srcs
   crypto/CryptoContextPool.cc
   crypto/CryptoImageDispatch.cc
   crypto/CryptoObjectDispatch.cc
+  crypto/FormatRequest.cc
+  crypto/LoadRequest.cc
   crypto/openssl/DataCryptor.cc
   deep_copy/ImageCopyRequest.cc
   deep_copy/MetadataCopyRequest.cc
@@ -209,6 +212,7 @@ endif()
 
 if(LINUX AND HAVE_LIBCRYPTSETUP)
   list(APPEND librbd_internal_srcs
+          crypto/luks/EncryptionFormat.cc
           crypto/luks/Header.cc
           crypto/luks/FormatRequest.cc
           crypto/luks/LoadRequest.cc)
index 99e00644b6d1a98fb28d7fedf6301db40889f65b..08e87c5deea7aa66b0647a75e0e047bd01f5ad5d 100644 (file)
@@ -534,6 +534,18 @@ librados::IoCtx duplicate_io_ctx(librados::IoCtx& io_ctx) {
     return 0;
   }
 
+  uint64_t ImageCtx::get_effective_image_size(snap_t in_snap_id) const {
+    auto raw_size = get_image_size(in_snap_id);
+    if (raw_size == 0) {
+      return 0;
+    }
+
+    io::Extents extents = {{raw_size, 0}};
+    io_image_dispatcher->remap_extents(
+            extents, io::IMAGE_EXTENTS_MAP_TYPE_PHYSICAL_TO_LOGICAL);
+    return extents.front().first;
+  }
+
   uint64_t ImageCtx::get_object_count(snap_t in_snap_id) const {
     ceph_assert(ceph_mutex_is_locked(image_lock));
     uint64_t image_size = get_image_size(in_snap_id);
index cc7a914a0e78e55d7d56fb3ae6c08ef3b4b54f5a..5623f2e768f6f57e5f560c29d1a1df12ae20a553 100644 (file)
@@ -301,6 +301,7 @@ namespace librbd {
                 std::string in_snap_name,
                 librados::snap_t id);
     uint64_t get_image_size(librados::snap_t in_snap_id) const;
+    uint64_t get_effective_image_size(librados::snap_t in_snap_id) const;
     uint64_t get_object_count(librados::snap_t in_snap_id) const;
     bool test_features(uint64_t test_features) const;
     bool test_features(uint64_t test_features,
index b68824d9d6f35ecdc6fc75e78a8f3f7728e3d840..b234d4f18f736975c45aac7a1b53c6c0b086fff6 100644 (file)
 #include "librbd/Utils.h"
 #include "librbd/api/Config.h"
 #include "librbd/api/Trash.h"
+#include "librbd/api/Utils.h"
+#include "librbd/crypto/FormatRequest.h"
+#include "librbd/crypto/LoadRequest.h"
 #include "librbd/deep_copy/Handler.h"
 #include "librbd/image/CloneRequest.h"
 #include "librbd/image/RemoveRequest.h"
 #include "librbd/image/PreRemoveRequest.h"
+#include "librbd/io/ImageDispatcherInterface.h"
+#include "librbd/io/ObjectDispatcherInterface.h"
 #include <boost/scope_exit.hpp>
 
 #define dout_subsys ceph_subsys_rbd
@@ -397,7 +402,8 @@ int Image<I>::list_descendants(
     }
 
     IoCtx ioctx;
-    r = util::create_ioctx(ictx->md_ctx, "child image", it.first, {}, &ioctx);
+    r = librbd::util::create_ioctx(
+            ictx->md_ctx, "child image", it.first, {}, &ioctx);
     if (r == -ENOENT) {
       continue;
     } else if (r < 0) {
@@ -425,16 +431,17 @@ int Image<I>::list_descendants(
 
   // retrieve clone v2 children attached to this snapshot
   IoCtx parent_io_ctx;
-  r = util::create_ioctx(ictx->md_ctx, "parent image", parent_spec.pool_id,
-                         parent_spec.pool_namespace, &parent_io_ctx);
+  r = librbd::util::create_ioctx(
+          ictx->md_ctx, "parent image",parent_spec.pool_id,
+          parent_spec.pool_namespace, &parent_io_ctx);
   if (r < 0) {
     return r;
   }
 
   cls::rbd::ChildImageSpecs child_images;
-  r = cls_client::children_list(&parent_io_ctx,
-                                util::header_name(parent_spec.image_id),
-                                parent_spec.snap_id, &child_images);
+  r = cls_client::children_list(
+          &parent_io_ctx, librbd::util::header_name(parent_spec.image_id),
+          parent_spec.snap_id, &child_images);
   if (r < 0 && r != -ENOENT && r != -EOPNOTSUPP) {
     lderr(cct) << "error retrieving children: " << cpp_strerror(r) << dendl;
     return r;
@@ -446,8 +453,9 @@ int Image<I>::list_descendants(
       child_image.image_id, "", false});
     if (!child_max_level || *child_max_level > 0) {
       IoCtx ioctx;
-      r = util::create_ioctx(ictx->md_ctx, "child image", child_image.pool_id,
-                             child_image.pool_namespace, &ioctx);
+      r = librbd::util::create_ioctx(
+              ictx->md_ctx, "child image", child_image.pool_id,
+              child_image.pool_namespace, &ioctx);
       if (r == -ENOENT) {
         continue;
       } else if (r < 0) {
@@ -470,8 +478,9 @@ int Image<I>::list_descendants(
   for (auto& image : *images) {
     if (child_pool_id == -1 || child_pool_id != image.pool_id ||
         child_io_ctx.get_namespace() != image.pool_namespace) {
-      r = util::create_ioctx(ictx->md_ctx, "child image", image.pool_id,
-                             image.pool_namespace, &child_io_ctx);
+      r = librbd::util::create_ioctx(
+              ictx->md_ctx, "child image", image.pool_id, image.pool_namespace,
+              &child_io_ctx);
       if (r == -ENOENT) {
         image.pool_name = "";
         image.image_name = "";
@@ -604,8 +613,9 @@ int Image<I>::deep_copy(I *src, librados::IoCtx& dest_md_ctx,
     r = create(dest_md_ctx, destname, "", src_size, opts, "", "", false);
   } else {
     librados::IoCtx parent_io_ctx;
-    r = util::create_ioctx(src->md_ctx, "parent image", parent_spec.pool_id,
-                           parent_spec.pool_namespace, &parent_io_ctx);
+    r = librbd::util::create_ioctx(
+            src->md_ctx, "parent image", parent_spec.pool_id,
+            parent_spec.pool_namespace, &parent_io_ctx);
     if (r < 0) {
       return r;
     }
@@ -614,7 +624,7 @@ int Image<I>::deep_copy(I *src, librados::IoCtx& dest_md_ctx,
     api::Config<I>::apply_pool_overrides(dest_md_ctx, &config);
 
     C_SaferCond ctx;
-    std::string dest_id = util::generate_image_id(dest_md_ctx);
+    std::string dest_id = librbd::util::generate_image_id(dest_md_ctx);
     auto *req = image::CloneRequest<I>::create(
       config, parent_io_ctx, parent_spec.image_id, "", {}, parent_spec.snap_id,
       dest_md_ctx, destname, dest_id, opts, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL,
@@ -874,9 +884,9 @@ int Image<I>::flatten_children(I *ictx, const char* snap_name,
     if (child_pool_id == -1 ||
         child_pool_id != child_image.pool_id ||
         child_io_ctx.get_namespace() != child_image.pool_namespace) {
-      r = util::create_ioctx(ictx->md_ctx, "child image",
-                             child_image.pool_id, child_image.pool_namespace,
-                             &child_io_ctx);
+      r = librbd::util::create_ioctx(
+              ictx->md_ctx, "child image", child_image.pool_id,
+              child_image.pool_namespace, &child_io_ctx);
       if (r < 0) {
         return r;
       }
@@ -924,6 +934,44 @@ int Image<I>::flatten_children(I *ictx, const char* snap_name,
   return 0;
 }
 
+template <typename I>
+int Image<I>::encryption_format(I* ictx, encryption_format_t format,
+                                encryption_options_t opts, size_t opts_size,
+                                bool c_api) {
+  crypto::EncryptionFormat<I>* result_format;
+  auto r = util::create_encryption_format(
+          ictx->cct, format, opts, opts_size, c_api, &result_format);
+  if (r != 0) {
+    return r;
+  }
+
+  C_SaferCond cond;
+  auto req = librbd::crypto::FormatRequest<I>::create(
+          ictx, std::unique_ptr<crypto::EncryptionFormat<I>>(result_format),
+          &cond);
+  req->send();
+  return cond.wait();
+}
+
+template <typename I>
+int Image<I>::encryption_load(I* ictx, encryption_format_t format,
+                              encryption_options_t opts, size_t opts_size,
+                              bool c_api) {
+  crypto::EncryptionFormat<I>* result_format;
+  auto r = util::create_encryption_format(
+          ictx->cct, format, opts, opts_size, c_api, &result_format);
+  if (r != 0) {
+    return r;
+  }
+
+  C_SaferCond cond;
+  auto req = librbd::crypto::LoadRequest<I>::create(
+          ictx, std::unique_ptr<crypto::EncryptionFormat<I>>(result_format),
+          &cond);
+  req->send();
+  return cond.wait();
+}
+
 } // namespace api
 } // namespace librbd
 
index 15a071de8f5b890e449abb61eb1cd3801fbf9f6c..192f9b7a79983750548e0f62d93448c0042f0b50 100644 (file)
@@ -70,6 +70,13 @@ struct Image {
 
   static int flatten_children(ImageCtxT *ictx, const char* snap_name, ProgressContext& pctx);
 
+  static int encryption_format(ImageCtxT *ictx, encryption_format_t format,
+                               encryption_options_t opts, size_t opts_size,
+                               bool c_api);
+  static int encryption_load(ImageCtxT *ictx, encryption_format_t format,
+                             encryption_options_t opts, size_t opts_size,
+                             bool c_api);
+
 };
 
 } // namespace api
diff --git a/src/librbd/api/Utils.cc b/src/librbd/api/Utils.cc
new file mode 100644 (file)
index 0000000..1ffb2f1
--- /dev/null
@@ -0,0 +1,84 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/api/Utils.h"
+#include "common/dout.h"
+
+#if defined(HAVE_LIBCRYPTSETUP)
+#include "librbd/crypto/luks/EncryptionFormat.h"
+#endif
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::api::util: " << __func__ << ": "
+
+namespace librbd {
+namespace api {
+namespace util {
+
+template <typename I>
+int create_encryption_format(
+        CephContext* cct, encryption_format_t format,
+        encryption_options_t opts, size_t opts_size, bool c_api,
+        crypto::EncryptionFormat<I>** result_format) {
+  size_t expected_opts_size;
+  switch (format) {
+#if defined(HAVE_LIBCRYPTSETUP)
+    case RBD_ENCRYPTION_FORMAT_LUKS1: {
+      if (c_api) {
+        expected_opts_size = sizeof(rbd_encryption_luks1_format_options_t);
+        if (expected_opts_size == opts_size) {
+          auto c_opts = (rbd_encryption_luks1_format_options_t*)opts;
+          *result_format = new crypto::luks::LUKS1EncryptionFormat<I>(
+                  c_opts->alg, {c_opts->passphrase, c_opts->passphrase_size});
+        }
+      } else {
+        expected_opts_size = sizeof(encryption_luks1_format_options_t);
+        if (expected_opts_size == opts_size) {
+          auto cpp_opts = (encryption_luks1_format_options_t*)opts;
+          *result_format = new crypto::luks::LUKS1EncryptionFormat<I>(
+                  cpp_opts->alg, std::move(cpp_opts->passphrase));
+        }
+      }
+      break;
+    }
+    case RBD_ENCRYPTION_FORMAT_LUKS2: {
+      if (c_api) {
+        expected_opts_size = sizeof(rbd_encryption_luks2_format_options_t);
+        if (expected_opts_size == opts_size) {
+          auto c_opts = (rbd_encryption_luks2_format_options_t*)opts;
+          *result_format = new crypto::luks::LUKS2EncryptionFormat<I>(
+                  c_opts->alg, {c_opts->passphrase, c_opts->passphrase_size});
+        }
+      } else {
+        expected_opts_size = sizeof(encryption_luks2_format_options_t);
+        if (expected_opts_size == opts_size) {
+          auto cpp_opts = (encryption_luks2_format_options_t*)opts;
+          *result_format = new crypto::luks::LUKS2EncryptionFormat<I>(
+                  cpp_opts->alg, std::move(cpp_opts->passphrase));
+        }
+      }
+      break;
+    }
+#endif
+    default:
+      lderr(cct) << "unsupported encryption format: " << format << dendl;
+      return -ENOTSUP;
+  }
+
+  if (expected_opts_size != opts_size) {
+    lderr(cct) << "expected opts_size: " << expected_opts_size << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+} // namespace util
+} // namespace api
+} // namespace librbd
+
+template int librbd::api::util::create_encryption_format(
+    CephContext* cct, encryption_format_t format, encryption_options_t opts,
+    size_t opts_size, bool c_api,
+    crypto::EncryptionFormat<librbd::ImageCtx>** result_format);
diff --git a/src/librbd/api/Utils.h b/src/librbd/api/Utils.h
new file mode 100644 (file)
index 0000000..8f8c222
--- /dev/null
@@ -0,0 +1,28 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_API_UTILS_H
+#define CEPH_LIBRBD_API_UTILS_H
+
+#include "include/rbd/librbd.hpp"
+#include "librbd/ImageCtx.h"
+#include "librbd/crypto/EncryptionFormat.h"
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace api {
+namespace util {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+int create_encryption_format(
+        CephContext* cct, encryption_format_t format,
+        encryption_options_t opts, size_t opts_size, bool c_api,
+        crypto::EncryptionFormat<ImageCtxT>** result_format);
+
+} // namespace util
+} // namespace api
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_API_UTILS_H
index 818efeb540446e43d44db4322605ca4f4497514b..dae3dac853b1dcd7ab225c1df3f61e82bad431fb 100644 (file)
@@ -11,6 +11,9 @@ namespace crypto {
 
 class CryptoImageDispatch : public io::ImageDispatchInterface {
 public:
+  static CryptoImageDispatch* create(uint64_t data_offset) {
+    return new CryptoImageDispatch(data_offset);
+  }
   CryptoImageDispatch(uint64_t data_offset);
 
   io::ImageDispatchLayer get_dispatch_layer() const override {
@@ -90,6 +93,10 @@ public:
     return false;
   }
 
+  bool invalidate_cache(Context* on_finish) override {
+    return false;
+  }
+
   void remap_extents(io::Extents& image_extents,
                      io::ImageExtentsMapType type) override;
 
index c2c24c8ce2f36efa9fd1c034aa3797547c15a38a..9cac7140761dab5d2ed6b4c4bca9afb257ebdd84 100644 (file)
@@ -25,9 +25,6 @@ namespace crypto {
 using librbd::util::create_context_callback;
 using librbd::util::data_object_name;
 
-io::ObjectDispatchLayer PREVIOUS_LAYER =
-        io::util::get_previous_layer(io::OBJECT_DISPATCH_LAYER_CRYPTO);
-
 template <typename I>
 struct C_AlignedObjectReadRequest : public Context {
     I* image_ctx;
@@ -122,8 +119,10 @@ struct C_UnalignedObjectReadRequest : public Context {
 
       // send the aligned read back to get decrypted
       req = io::ObjectDispatchSpec::create_read(
-              image_ctx, PREVIOUS_LAYER, object_no, &aligned_extents,
-              io_context, op_flags, read_flags, parent_trace, version, this);
+              image_ctx,
+              io::util::get_previous_layer(io::OBJECT_DISPATCH_LAYER_CRYPTO),
+              object_no, &aligned_extents, io_context, op_flags, read_flags,
+              parent_trace, version, this);
     }
 
     void send() {
@@ -381,9 +380,10 @@ struct C_UnalignedObjectWriteRequest : public Context {
 
       // send back aligned write back to get encrypted and committed
       auto write_req = io::ObjectDispatchSpec::create_write(
-              image_ctx, PREVIOUS_LAYER, object_no, aligned_off,
-              std::move(aligned_data), io_context, op_flags, new_write_flags,
-              new_assert_version,
+              image_ctx,
+              io::util::get_previous_layer(io::OBJECT_DISPATCH_LAYER_CRYPTO),
+              object_no, aligned_off, std::move(aligned_data), io_context,
+              op_flags, new_write_flags, new_assert_version,
               journal_tid == nullptr ? 0 : *journal_tid, parent_trace, ctx);
       write_req->send();
     }
@@ -427,18 +427,6 @@ CryptoObjectDispatch<I>::CryptoObjectDispatch(
   : m_image_ctx(image_ctx), m_crypto(crypto) {
 }
 
-template <typename I>
-void CryptoObjectDispatch<I>::init(Context* on_finish) {
-  auto cct = m_image_ctx->cct;
-  ldout(cct, 5) << dendl;
-
-  // need to initialize m_crypto here using image header object
-
-  m_image_ctx->io_object_dispatcher->register_dispatch(this);
-
-  on_finish->complete(0);
-}
-
 template <typename I>
 void CryptoObjectDispatch<I>::shut_down(Context* on_finish) {
   if (m_crypto != nullptr) {
@@ -537,9 +525,10 @@ bool CryptoObjectDispatch<I>::write_same(
 
   *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
   auto req = io::ObjectDispatchSpec::create_write(
-          m_image_ctx, PREVIOUS_LAYER, object_no, object_off,
-          std::move(ws_data), io_context, op_flags, 0, std::nullopt, 0,
-          parent_trace, ctx);
+          m_image_ctx,
+          io::util::get_previous_layer(io::OBJECT_DISPATCH_LAYER_CRYPTO),
+          object_no, object_off, std::move(ws_data), io_context, op_flags, 0,
+          std::nullopt, 0, parent_trace, ctx);
   req->send();
   return true;
 }
@@ -593,9 +582,10 @@ bool CryptoObjectDispatch<I>::discard(
 
   *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
   auto req = io::ObjectDispatchSpec::create_write_same(
-          m_image_ctx, PREVIOUS_LAYER, object_no, object_off, object_len,
-          {{0, object_len}}, std::move(bl), io_context,
-          *object_dispatch_flags, 0, parent_trace, ctx);
+          m_image_ctx,
+          io::util::get_previous_layer(io::OBJECT_DISPATCH_LAYER_CRYPTO),
+          object_no, object_off, object_len, {{0, object_len}}, std::move(bl),
+          io_context, *object_dispatch_flags, 0, parent_trace, ctx);
   req->send();
   return true;
 }
index 6df32ffee6d36c84d7e371a694e05c24194f85d0..1c5a4646d087945bcc6f64f615f1a4da8cb4619e 100644 (file)
@@ -17,8 +17,9 @@ namespace crypto {
 template <typename ImageCtxT = librbd::ImageCtx>
 class CryptoObjectDispatch : public io::ObjectDispatchInterface {
 public:
-  static CryptoObjectDispatch* create(ImageCtxT* image_ctx) {
-    return new CryptoObjectDispatch(image_ctx, nullptr);
+  static CryptoObjectDispatch* create(
+          ImageCtxT* image_ctx, ceph::ref_t<CryptoInterface> crypto) {
+    return new CryptoObjectDispatch(image_ctx, crypto);
   }
 
   CryptoObjectDispatch(ImageCtxT* image_ctx,
@@ -28,8 +29,6 @@ public:
     return io::OBJECT_DISPATCH_LAYER_CRYPTO;
   }
 
-  void init(Context* on_finish);
-
   void shut_down(Context* on_finish) override;
 
   bool read(
@@ -103,7 +102,6 @@ public:
       io::SnapshotSparseBufferlist* snapshot_sparse_bufferlist) override;
 
 private:
-
   ImageCtxT* m_image_ctx;
   ceph::ref_t<CryptoInterface> m_crypto;
 
diff --git a/src/librbd/crypto/EncryptionFormat.h b/src/librbd/crypto/EncryptionFormat.h
new file mode 100644 (file)
index 0000000..1ac9622
--- /dev/null
@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CRYPTO_ENCRYPTION_FORMAT_H
+#define CEPH_LIBRBD_CRYPTO_ENCRYPTION_FORMAT_H
+
+#include "common/ref.h"
+
+struct Context;
+
+namespace librbd {
+namespace crypto {
+
+struct CryptoInterface;
+
+template <typename ImageCtxT>
+struct EncryptionFormat {
+  virtual ~EncryptionFormat() {
+  }
+
+  virtual void format(ImageCtxT* ictx, Context* on_finish) = 0;
+  virtual void load(ImageCtxT* ictx,
+                    ceph::ref_t<CryptoInterface>* result_crypto,
+                    Context* on_finish) = 0;
+};
+
+} // namespace crypto
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_CRYPTO_ENCRYPTION_FORMAT_H
diff --git a/src/librbd/crypto/FormatRequest.cc b/src/librbd/crypto/FormatRequest.cc
new file mode 100644 (file)
index 0000000..d23f950
--- /dev/null
@@ -0,0 +1,66 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "FormatRequest.h"
+
+#include "common/dout.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/io/ObjectDispatcherInterface.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::crypto::FormatRequest: " << this \
+                           << " " << __func__ << ": "
+
+namespace librbd {
+namespace crypto {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+FormatRequest<I>::FormatRequest(
+        I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+        Context* on_finish) : m_image_ctx(image_ctx),
+                              m_format(std::move(format)),
+                              m_on_finish(on_finish) {
+}
+
+template <typename I>
+void FormatRequest<I>::send() {
+  if (m_image_ctx->io_object_dispatcher->exists(
+          io::OBJECT_DISPATCH_LAYER_CRYPTO)) {
+    lderr(m_image_ctx->cct) << "cannot format with already loaded encryption"
+                            << dendl;
+    finish(-EEXIST);
+    return;
+  }
+
+  if (m_image_ctx->parent != nullptr) {
+    lderr(m_image_ctx->cct) << "cannot format a cloned image" << dendl;
+    finish(-ENOTSUP);
+    return;
+  }
+
+  if (m_image_ctx->test_features(RBD_FEATURE_JOURNALING)) {
+    lderr(m_image_ctx->cct) << "cannot use encryption with journal" << dendl;
+    finish(-ENOTSUP);
+    return;
+  }
+
+  auto ctx = create_context_callback<
+          FormatRequest<I>, &FormatRequest<I>::finish>(this);
+  m_format->format(m_image_ctx, ctx);
+}
+
+template <typename I>
+void FormatRequest<I>::finish(int r) {
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace crypto
+} // namespace librbd
+
+template class librbd::crypto::FormatRequest<librbd::ImageCtx>;
diff --git a/src/librbd/crypto/FormatRequest.h b/src/librbd/crypto/FormatRequest.h
new file mode 100644 (file)
index 0000000..fc4263a
--- /dev/null
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CRYPTO_FORMAT_REQUEST_H
+#define CEPH_LIBRBD_CRYPTO_FORMAT_REQUEST_H
+
+#include "include/rbd/librbd.hpp"
+#include "librbd/crypto/EncryptionFormat.h"
+
+struct Context;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace crypto {
+
+template <typename I>
+class FormatRequest {
+public:
+    static FormatRequest* create(
+            I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+            Context* on_finish) {
+      return new FormatRequest(image_ctx, std::move(format), on_finish);
+    }
+
+    FormatRequest(I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+                  Context* on_finish);
+    void send();
+    void finish(int r);
+
+private:
+    I* m_image_ctx;
+
+    std::unique_ptr<EncryptionFormat<I>> m_format;
+    Context* m_on_finish;
+};
+
+} // namespace crypto
+} // namespace librbd
+
+extern template class librbd::crypto::FormatRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CRYPTO_FORMAT_REQUEST_H
diff --git a/src/librbd/crypto/LoadRequest.cc b/src/librbd/crypto/LoadRequest.cc
new file mode 100644 (file)
index 0000000..2e5cb02
--- /dev/null
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "LoadRequest.h"
+
+#include "common/dout.h"
+#include "common/errno.h"
+#include "librbd/Utils.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/crypto/CryptoImageDispatch.h"
+#include "librbd/crypto/CryptoObjectDispatch.h"
+#include "librbd/io/ImageDispatcherInterface.h"
+#include "librbd/io/ObjectDispatcherInterface.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::crypto::LoadRequest: " << this \
+                           << " " << __func__ << ": "
+
+namespace librbd {
+namespace crypto {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+LoadRequest<I>::LoadRequest(
+        I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+        Context* on_finish) : m_image_ctx(image_ctx),
+                              m_format(std::move(format)),
+                              m_on_finish(on_finish) {
+}
+
+template <typename I>
+void LoadRequest<I>::send() {
+  if (m_image_ctx->io_object_dispatcher->exists(
+          io::OBJECT_DISPATCH_LAYER_CRYPTO)) {
+    lderr(m_image_ctx->cct) << "encryption already loaded" << dendl;
+    finish(-EEXIST);
+    return;
+  }
+
+  auto ictx = m_image_ctx;
+  while (ictx != nullptr) {
+    if (ictx->test_features(RBD_FEATURE_JOURNALING)) {
+      lderr(m_image_ctx->cct) << "cannot use encryption with journal."
+                              << " image name: " << ictx->name << dendl;
+      finish(-ENOTSUP);
+      return;
+    }
+    ictx = ictx->parent;
+  }
+
+  auto ctx = create_context_callback<
+          LoadRequest<I>, &LoadRequest<I>::finish>(this);
+  m_format->load(m_image_ctx, &m_crypto, ctx);
+}
+
+template <typename I>
+void LoadRequest<I>::finish(int r) {
+  if (r == 0) {
+    // load crypto layers to image and its ancestors
+    auto image_dispatch = CryptoImageDispatch::create(
+            m_crypto->get_data_offset());
+    auto ictx = m_image_ctx;
+    while (ictx != nullptr) {
+      auto object_dispatch = CryptoObjectDispatch<I>::create(
+              ictx, m_crypto);
+      ictx->io_object_dispatcher->register_dispatch(object_dispatch);
+      ictx->io_image_dispatcher->register_dispatch(image_dispatch);
+
+      ictx = ictx->parent;
+    }
+  }
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace crypto
+} // namespace librbd
+
+template class librbd::crypto::LoadRequest<librbd::ImageCtx>;
diff --git a/src/librbd/crypto/LoadRequest.h b/src/librbd/crypto/LoadRequest.h
new file mode 100644 (file)
index 0000000..6bbe980
--- /dev/null
@@ -0,0 +1,45 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CRYPTO_LOAD_REQUEST_H
+#define CEPH_LIBRBD_CRYPTO_LOAD_REQUEST_H
+
+#include "include/rbd/librbd.hpp"
+#include "librbd/crypto/CryptoInterface.h"
+#include "librbd/crypto/EncryptionFormat.h"
+
+struct Context;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace crypto {
+
+template <typename I>
+class LoadRequest {
+public:
+    static LoadRequest* create(
+            I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+            Context* on_finish) {
+      return new LoadRequest(image_ctx, std::move(format), on_finish);
+    }
+
+    LoadRequest(I* image_ctx, std::unique_ptr<EncryptionFormat<I>> format,
+                Context* on_finish);
+    void send();
+    void finish(int r);
+
+private:
+    I* m_image_ctx;
+    std::unique_ptr<EncryptionFormat<I>> m_format;
+    Context* m_on_finish;
+    ceph::ref_t<crypto::CryptoInterface> m_crypto;
+};
+
+} // namespace crypto
+} // namespace librbd
+
+extern template class librbd::crypto::LoadRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CRYPTO_LOAD_REQUEST_H
index c7b29d687dd716ec06b3f4b37637b263958173f4..93d9c172c06223b34965054410433fbb13dd376f 100644 (file)
@@ -12,16 +12,6 @@ enum CipherMode {
     CIPHER_MODE_DEC,
 };
 
-enum DiskEncryptionFormat {
-    DISK_ENCRYPTION_FORMAT_LUKS1,
-    DISK_ENCRYPTION_FORMAT_LUKS2,
-};
-
-enum CipherAlgorithm {
-    CIPHER_ALGORITHM_AES128,
-    CIPHER_ALGORITHM_AES256,
-};
-
 } // namespace crypto
 } // namespace librbd
 
diff --git a/src/librbd/crypto/luks/EncryptionFormat.cc b/src/librbd/crypto/luks/EncryptionFormat.cc
new file mode 100644 (file)
index 0000000..d89ef36
--- /dev/null
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "EncryptionFormat.h"
+#include "librbd/crypto/luks/FormatRequest.h"
+#include "librbd/crypto/luks/LoadRequest.h"
+
+namespace librbd {
+namespace crypto {
+namespace luks {
+
+template <typename I>
+EncryptionFormat<I>::EncryptionFormat(
+        encryption_algorithm_t alg,
+        std::string&& passphrase) : m_alg(alg),
+                                    m_passphrase(std::move(passphrase)) {
+}
+
+template <typename I>
+void EncryptionFormat<I>::format(I* image_ctx, Context* on_finish) {
+  auto req = luks::FormatRequest<I>::create(
+          image_ctx, get_format(), m_alg, std::move(m_passphrase), on_finish,
+          false);
+  req->send();
+}
+
+template <typename I>
+void EncryptionFormat<I>::load(
+        I* image_ctx, ceph::ref_t<CryptoInterface>* result_crypto,
+        Context* on_finish) {
+  auto req = luks::LoadRequest<I>::create(
+          image_ctx, get_format(), std::move(m_passphrase), result_crypto,
+          on_finish);
+  req->send();
+}
+
+} // namespace luks
+} // namespace crypto
+} // namespace librbd
+
+template class librbd::crypto::luks::EncryptionFormat<librbd::ImageCtx>;
+template class librbd::crypto::luks::LUKS1EncryptionFormat<librbd::ImageCtx>;
+template class librbd::crypto::luks::LUKS2EncryptionFormat<librbd::ImageCtx>;
diff --git a/src/librbd/crypto/luks/EncryptionFormat.h b/src/librbd/crypto/luks/EncryptionFormat.h
new file mode 100644 (file)
index 0000000..a45a3c9
--- /dev/null
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CRYPTO_LUKS_ENCRYPTION_FORMAT_H
+#define CEPH_LIBRBD_CRYPTO_LUKS_ENCRYPTION_FORMAT_H
+
+#include "include/rbd/librbd.hpp"
+#include "librbd/crypto/EncryptionFormat.h"
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace crypto {
+namespace luks {
+
+template <typename ImageCtxT>
+class EncryptionFormat : public crypto::EncryptionFormat<ImageCtxT> {
+
+public:
+    EncryptionFormat(encryption_algorithm_t alg, std::string&& passphrase);
+
+    void format(ImageCtxT* ictx, Context* on_finish) override;
+    void load(ImageCtxT* ictx, ceph::ref_t<CryptoInterface>* result_crypto,
+              Context* on_finish) override;
+
+private:
+    virtual encryption_format_t get_format() = 0;
+
+    encryption_algorithm_t m_alg;
+    std::string m_passphrase;
+};
+
+template <typename ImageCtxT>
+class LUKS1EncryptionFormat : public EncryptionFormat<ImageCtxT> {
+    using EncryptionFormat<ImageCtxT>::EncryptionFormat;
+
+    encryption_format_t get_format() override {
+      return RBD_ENCRYPTION_FORMAT_LUKS1;
+    }
+};
+
+template <typename ImageCtxT>
+class LUKS2EncryptionFormat : public EncryptionFormat<ImageCtxT> {
+    using EncryptionFormat<ImageCtxT>::EncryptionFormat;
+
+    encryption_format_t get_format() override {
+      return RBD_ENCRYPTION_FORMAT_LUKS2;
+    }
+};
+
+} // namespace luks
+} // namespace crypto
+} // namespace librbd
+
+extern template class librbd::crypto::luks::EncryptionFormat<librbd::ImageCtx>;
+extern template class librbd::crypto::luks::LUKS1EncryptionFormat<
+        librbd::ImageCtx>;
+extern template class librbd::crypto::luks::LUKS2EncryptionFormat<
+        librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CRYPTO_LUKS_ENCRYPTION_FORMAT_H
index 205c118d779f13b9aa048b48cdb7645635c8f569..b383d12ca887afad0c17b046b75d1fc5a574fe1f 100644 (file)
@@ -9,7 +9,6 @@
 #include "librbd/crypto/luks/Header.h"
 #include "librbd/io/AioCompletion.h"
 #include "librbd/io/ImageDispatchSpec.h"
-#include "librbd/io/ObjectDispatcherInterface.h"
 
 #define dout_subsys ceph_subsys_rbd
 #undef dout_prefix
@@ -24,54 +23,49 @@ using librbd::util::create_context_callback;
 
 template <typename I>
 FormatRequest<I>::FormatRequest(
-        I* image_ctx, DiskEncryptionFormat type, CipherAlgorithm cipher,
+        I* image_ctx, encryption_format_t format, encryption_algorithm_t alg,
         std::string&& passphrase, Context* on_finish,
-        bool insecure_fast_mode) : m_image_ctx(image_ctx), m_type(type),
-                                   m_cipher(cipher), m_on_finish(on_finish),
+        bool insecure_fast_mode) : m_image_ctx(image_ctx), m_format(format),
+                                   m_alg(alg),
+                                   m_passphrase(std::move(passphrase)),
+                                   m_on_finish(on_finish),
                                    m_insecure_fast_mode(insecure_fast_mode),
-                                   m_header(image_ctx->cct),
-                                   m_passphrase(std::move(passphrase)) {
+                                   m_header(image_ctx->cct) {
 }
 
 template <typename I>
 void FormatRequest<I>::send() {
-  if (m_image_ctx->io_object_dispatcher->exists(
-          io::OBJECT_DISPATCH_LAYER_CRYPTO)) {
-    finish(-EEXIST);
-    return;
-  }
-
   const char* type;
   size_t sector_size;
-  switch(m_type) {
-    case DISK_ENCRYPTION_FORMAT_LUKS1:
+  switch (m_format) {
+    case RBD_ENCRYPTION_FORMAT_LUKS1:
       type = CRYPT_LUKS1;
       sector_size = 512;
       break;
-    case DISK_ENCRYPTION_FORMAT_LUKS2:
+    case RBD_ENCRYPTION_FORMAT_LUKS2:
       type = CRYPT_LUKS2;
       sector_size = 4096;
       break;
     default:
-      lderr(m_image_ctx->cct) << "unsupported disk encryption type: " << m_type
+      lderr(m_image_ctx->cct) << "unsupported format type: " << m_format
                               << dendl;
       finish(-EINVAL);
       return;
   }
 
-  const char* alg;
+  const char* cipher;
   size_t key_size;
-  switch (m_cipher) {
-    case CIPHER_ALGORITHM_AES128:
-      alg = "aes";
+  switch (m_alg) {
+    case RBD_ENCRYPTION_ALGORITHM_AES128:
+      cipher = "aes";
       key_size = 32;
       break;
-    case CIPHER_ALGORITHM_AES256:
-      alg = "aes";
+    case RBD_ENCRYPTION_ALGORITHM_AES256:
+      cipher = "aes";
       key_size = 64;
       break;
     default:
-      lderr(m_image_ctx->cct) << "unsupported cipher algorithm: " << m_cipher
+      lderr(m_image_ctx->cct) << "unsupported cipher algorithm: " << m_alg
                               << dendl;
       finish(-EINVAL);
       return;
@@ -85,7 +79,7 @@ void FormatRequest<I>::send() {
   }
 
   // format (create LUKS header)
-  r = m_header.format(type, alg, key_size, "xts-plain64", sector_size,
+  r = m_header.format(type, cipher, key_size, "xts-plain64", sector_size,
                       m_image_ctx->get_object_size(), m_insecure_fast_mode);
   if (r != 0) {
     finish(r);
index 3574ed39aee137caef165fde62bbda098373d9c1..1723185422da417615e915172878c20d67c59d3e 100644 (file)
@@ -4,8 +4,8 @@
 #ifndef CEPH_LIBRBD_CRYPTO_LUKS_FORMAT_REQUEST_H
 #define CEPH_LIBRBD_CRYPTO_LUKS_FORMAT_REQUEST_H
 
+#include "include/rbd/librbd.hpp"
 #include "librbd/ImageCtx.h"
-#include "librbd/crypto/Types.h"
 #include "librbd/crypto/luks/Header.h"
 
 namespace librbd {
@@ -19,15 +19,15 @@ template <typename I>
 class FormatRequest {
 public:
     static FormatRequest* create(
-            I* image_ctx, DiskEncryptionFormat type, CipherAlgorithm cipher,
-            std::string&& passphrase, Context* on_finish,
-            bool insecure_fast_mode) {
-      return new FormatRequest(image_ctx, type, cipher, std::move(passphrase),
+            I* image_ctx, encryption_format_t format,
+            encryption_algorithm_t alg, std::string&& passphrase,
+            Context* on_finish, bool insecure_fast_mode) {
+      return new FormatRequest(image_ctx, format, alg, std::move(passphrase),
                                on_finish, insecure_fast_mode);
     }
 
-    FormatRequest(I* image_ctx, DiskEncryptionFormat type,
-                  CipherAlgorithm cipher, std::string&& passphrase,
+    FormatRequest(I* image_ctx, encryption_format_t format,
+                  encryption_algorithm_t alg, std::string&& passphrase,
                   Context* on_finish, bool insecure_fast_mode);
     void send();
     void finish(int r);
@@ -35,12 +35,12 @@ public:
 private:
     I* m_image_ctx;
 
-    DiskEncryptionFormat m_type;
-    CipherAlgorithm m_cipher;
+    encryption_format_t m_format;
+    encryption_algorithm_t m_alg;
+    std::string m_passphrase;
     Context* m_on_finish;
     bool m_insecure_fast_mode;
     Header m_header;
-    std::string m_passphrase;
 
     void handle_write_header(int r);
 };
index 3dbc7d3b0ad1231b92959b805fd1570887016328..4307844f50467bec19898b2042d8b9eae1847d2a 100644 (file)
@@ -109,6 +109,8 @@ ssize_t Header::read(ceph::bufferlist* bl) {
   if (r < 0) {
     lderr(m_cct) << "error reading header: " << cpp_strerror(r) << dendl;
   }
+
+  ldout(m_cct, 20) << "read size = " << r << dendl;
   return r;
 }
 
@@ -117,6 +119,9 @@ int Header::format(const char* type, const char* alg, size_t key_size,
                    uint32_t data_alignment, bool insecure_fast_mode) {
   ceph_assert(m_cd != nullptr);
 
+  ldout(m_cct, 20) << "sector size: " << sector_size << ", data alignment: "
+                   << data_alignment << dendl;
+
   // required for passing libcryptsetup device size check
   if (ftruncate(m_fd, 4096) != 0) {
     lderr(m_cct) << "failed to truncate anonymous file: "
@@ -133,7 +138,6 @@ int Header::format(const char* type, const char* alg, size_t key_size,
   size_t converted_data_alignment = data_alignment / 512;
 #endif
 
-
   void* params = nullptr;
   if (strcmp(type, CRYPT_LUKS1) == 0) {
     memset(&luks1params, 0, sizeof(luks1params));
@@ -187,7 +191,7 @@ int Header::add_keyslot(const char* passphrase, size_t passphrase_size) {
   return 0;
 }
 
-int Header::load() {
+int Header::load(const char* type) {
   ceph_assert(m_cd != nullptr);
 
   // libcryptsetup checks if device size matches the header and keyslots size
@@ -198,12 +202,15 @@ int Header::load() {
     return -errno;
   }
 
-  auto r = crypt_load(m_cd, CRYPT_LUKS, NULL);
+  auto r = crypt_load(m_cd, type, NULL);
   if (r != 0) {
     lderr(m_cct) << "crypt_load failed: " << cpp_strerror(r) << dendl;
     return r;
   }
 
+  ldout(m_cct, 20) << "sector size: " << get_sector_size() << ", data offset: "
+                   << get_data_offset() << dendl;
+
   return 0;
 }
 
index 6b58f246423fc9e12e8d0ed1b540d25ff4350c03..13ef8fd20d31be3e3ae59f1a521a0cebaef51955 100644 (file)
@@ -25,7 +25,7 @@ public:
                const char* cipher_mode, uint32_t sector_size,
                uint32_t data_alignment, bool insecure_fast_mode);
     int add_keyslot(const char* passphrase, size_t passphrase_size);
-    int load();
+    int load(const char* type);
     int read_volume_key(const char* passphrase, size_t passphrase_size,
                         char* volume_key, size_t* volume_key_size);
 
index 5229b6e5903099676e57e2c1253c47766d0141c3..2b0dbe47d25dfe7ca1e536ab3dad1a2656bcec60 100644 (file)
@@ -10,7 +10,6 @@
 #include "librbd/crypto/openssl/DataCryptor.h"
 #include "librbd/io/AioCompletion.h"
 #include "librbd/io/ImageDispatchSpec.h"
-#include "librbd/io/ObjectDispatcherInterface.h"
 #include "librbd/io/ReadResult.h"
 
 #define dout_subsys ceph_subsys_rbd
@@ -26,13 +25,15 @@ using librbd::util::create_context_callback;
 
 template <typename I>
 LoadRequest<I>::LoadRequest(
-        I* image_ctx, std::string&& passphrase,
+        I* image_ctx, encryption_format_t format, std::string&& passphrase,
         ceph::ref_t<CryptoInterface>* result_crypto,
-        Context* on_finish) : m_image_ctx(image_ctx), m_on_finish(on_finish),
+        Context* on_finish) : m_image_ctx(image_ctx),
+                              m_format(format),
+                              m_passphrase(std::move(passphrase)),
+                              m_on_finish(on_finish),
                               m_result_crypto(result_crypto),
                               m_initial_read_size(DEFAULT_INITIAL_READ_SIZE),
-                              m_header(image_ctx->cct), m_offset(0),
-                              m_passphrase(std::move(passphrase)) {
+                              m_header(image_ctx->cct), m_offset(0) {
 }
 
 template <typename I>
@@ -42,12 +43,6 @@ void LoadRequest<I>::set_initial_read_size(uint64_t read_size) {
 
 template <typename I>
 void LoadRequest<I>::send() {
-  if (m_image_ctx->io_object_dispatcher->exists(
-          io::OBJECT_DISPATCH_LAYER_CRYPTO)) {
-    finish(-EEXIST);
-    return;
-  }
-
   // setup interface with libcryptsetup
   auto r = m_header.init();
   if (r < 0) {
@@ -100,8 +95,23 @@ void LoadRequest<I>::handle_read_header(int r) {
     return;
   }
 
+  const char* type;
+  switch (m_format) {
+    case RBD_ENCRYPTION_FORMAT_LUKS1:
+      type = CRYPT_LUKS1;
+      break;
+    case RBD_ENCRYPTION_FORMAT_LUKS2:
+      type = CRYPT_LUKS2;
+      break;
+    default:
+      lderr(m_image_ctx->cct) << "unsupported format type: " << m_format
+                              << dendl;
+      finish(-EINVAL);
+      return;
+  }
+
   // parse header via libcryptsetup
-  r = m_header.load();
+  r = m_header.load(type);
   if (r != 0) {
     if (m_offset < MAXIMUM_HEADER_SIZE) {
       // perhaps we did not feed the entire header to libcryptsetup, retry
index 4c7dca7bc3617d11a14670b6ab65ef0ae2e7675b..340e89503c0f8ca5b15cf1bd567ab963c313490f 100644 (file)
@@ -4,6 +4,7 @@
 #ifndef CEPH_LIBRBD_CRYPTO_LUKS_LOAD_REQUEST_H
 #define CEPH_LIBRBD_CRYPTO_LUKS_LOAD_REQUEST_H
 
+#include "include/rbd/librbd.hpp"
 #include "librbd/ImageCtx.h"
 #include "librbd/crypto/CryptoInterface.h"
 #include "librbd/crypto/luks/Header.h"
@@ -24,13 +25,14 @@ template <typename I>
 class LoadRequest {
 public:
     static LoadRequest* create(
-            I* image_ctx, std::string&& passphrase,
+            I* image_ctx, encryption_format_t format, std::string&& passphrase,
             ceph::ref_t<CryptoInterface>* result_crypto, Context* on_finish) {
-      return new LoadRequest(image_ctx, std::move(passphrase), result_crypto,
-                             on_finish);
+      return new LoadRequest(image_ctx, format, std::move(passphrase),
+                             result_crypto, on_finish);
     }
 
-    LoadRequest(I* image_ctx, std::string&& passphrase,
+    LoadRequest(I* image_ctx, encryption_format_t format,
+                std::string&& passphrase,
                 ceph::ref_t<CryptoInterface>* result_crypto,
                 Context* on_finish);
     void send();
@@ -39,13 +41,14 @@ public:
 
 private:
     I* m_image_ctx;
+    encryption_format_t m_format;
+    std::string m_passphrase;
     Context* m_on_finish;
     ceph::bufferlist m_bl;
     ceph::ref_t<CryptoInterface>* m_result_crypto;
     uint64_t m_initial_read_size;
     Header m_header;
     uint64_t m_offset;
-    std::string m_passphrase;
 
     void read(uint64_t end_offset, Context* on_finish);
     bool handle_read(int r);
index 46dabacc89025d5ebf5e400aeb414f2a7071c71b..db780956b561911c3f5f584a2ef139287008c97c 100644 (file)
@@ -181,7 +181,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) {
     int obj_order = ictx->order;
     {
       std::shared_lock locker{ictx->image_lock};
-      info.size = ictx->get_image_size(ictx->snap_id);
+      info.size = ictx->get_effective_image_size(ictx->snap_id);
     }
     info.obj_size = 1ULL << obj_order;
     info.num_objs = Striper::get_num_objects(ictx->layout, info.size);
@@ -858,7 +858,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) {
     if (r < 0)
       return r;
     std::shared_lock l2{ictx->image_lock};
-    *size = ictx->get_image_size(ictx->snap_id);
+    *size = ictx->get_effective_image_size(ictx->snap_id);
     return 0;
   }
 
@@ -1576,16 +1576,11 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) {
   {
     ceph_assert(ceph_mutex_is_locked(ictx->image_lock));
 
-    uint64_t image_size;
-    if (ictx->snap_id == CEPH_NOSNAP) {
-      image_size = ictx->get_image_size(CEPH_NOSNAP);
-    } else {
-      auto snap_info = ictx->get_snap_info(ictx->snap_id);
-      if (snap_info == nullptr) {
-       return -ENOENT;
-      }
-      image_size = snap_info->size;
+    if (ictx->snap_id != CEPH_NOSNAP &&
+        ictx->get_snap_info(ictx->snap_id) == nullptr) {
+      return -ENOENT;
     }
+    uint64_t image_size = ictx->get_effective_image_size(ictx->snap_id);
 
     // special-case "len == 0" requests: always valid
     if (*len == 0)
index 4b31f2c7ae8a9468b8da3251c60b66565a621802..2b4212df9decffd6eac69279a5ad6ddd467835fd 100644 (file)
@@ -145,7 +145,7 @@ void readahead(I *ictx, const Extents& image_extents, IOContext io_context) {
     return;
   }
 
-  uint64_t image_size = ictx->get_image_size(ictx->snap_id);
+  uint64_t image_size = ictx->get_effective_image_size(ictx->snap_id);
   ictx->image_lock.unlock_shared();
 
   auto readahead_extent = ictx->readahead.update(image_extents, image_size);
index 6d5956453fad8f75188bec7a212a9dc64718c0cf..f099143d24600ce33981a34eec54c338570442d0 100644 (file)
@@ -2045,6 +2045,24 @@ namespace librbd {
     return r;
   }
 
+  int Image::encryption_format(encryption_format_t format,
+                               encryption_options_t opts,
+                               size_t opts_size)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::api::Image<>::encryption_format(
+            ictx, format, opts, opts_size, false);
+  }
+
+  int Image::encryption_load(encryption_format_t format,
+                             encryption_options_t opts,
+                             size_t opts_size)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::api::Image<>::encryption_load(
+            ictx, format, opts, opts_size, false);
+  }
+
   int Image::flatten()
   {
     ImageCtx *ictx = (ImageCtx *)ctx;
@@ -4314,6 +4332,26 @@ extern "C" int rbd_deep_copy_with_progress(rbd_image_t image,
   return ret;
 }
 
+extern "C" int rbd_encryption_format(rbd_image_t image,
+                                     rbd_encryption_format_t format,
+                                     rbd_encryption_options_t opts,
+                                     size_t opts_size)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::api::Image<>::encryption_format(
+          ictx, format, opts, opts_size, true);
+}
+
+extern "C" int rbd_encryption_load(rbd_image_t image,
+                                   rbd_encryption_format_t format,
+                                   rbd_encryption_options_t opts,
+                                   size_t opts_size)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::api::Image<>::encryption_load(
+          ictx, format, opts, opts_size, true);
+}
+
 extern "C" int rbd_flatten(rbd_image_t image)
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
index 790d74fc1295dacfbd547fc2036fc08d69f4790d..e0d3775279e35e60122160a37655581df1ba8f4b 100644 (file)
@@ -56,6 +56,8 @@ set(unittest_librbd_srcs
   crypto/test_mock_BlockCrypto.cc
   crypto/test_mock_CryptoContextPool.cc
   crypto/test_mock_CryptoObjectDispatch.cc
+  crypto/test_mock_FormatRequest.cc
+  crypto/test_mock_LoadRequest.cc
   crypto/openssl/test_DataCryptor.cc
   deep_copy/test_mock_ImageCopyRequest.cc
   deep_copy/test_mock_MetadataCopyRequest.cc
index 0b7c71e262918a619b0c4a71122d46e940692098..8987163660993f661d9bdcd371ffc3c69b8f173a 100644 (file)
@@ -57,11 +57,6 @@ struct TestMockCryptoLuksFormatRequest : public TestMockFixture {
             OBJECT_SIZE));
   }
 
-  void expect_crypto_layer_exists_check(bool exists = false) {
-    EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, exists(
-            io::OBJECT_DISPATCH_LAYER_CRYPTO)).WillOnce(Return(exists));
-  }
-                         
   void expect_get_image_size(uint64_t image_size) {
     EXPECT_CALL(*mock_image_ctx, get_image_size(CEPH_NOSNAP)).WillOnce(Return(
             image_size));
@@ -94,13 +89,13 @@ struct TestMockCryptoLuksFormatRequest : public TestMockFixture {
     }
   }
 
-  void verify_header(size_t expected_key_length,
+  void verify_header(const char* expected_format, size_t expected_key_length,
                      uint64_t expected_sector_size) {
     Header header(mock_image_ctx->cct);
 
     ASSERT_EQ(0, header.init());
     ASSERT_EQ(0, header.write(header_bl));
-    ASSERT_EQ(0, header.load());
+    ASSERT_EQ(0, header.load(expected_format));
 
     ASSERT_EQ(expected_sector_size, header.get_sector_size());
     ASSERT_EQ(0, header.get_data_offset() % OBJECT_SIZE);
@@ -115,10 +110,9 @@ struct TestMockCryptoLuksFormatRequest : public TestMockFixture {
 
 TEST_F(TestMockCryptoLuksFormatRequest, LUKS1) {
   auto mock_format_request = MockFormatRequest::create(
-          mock_image_ctx, DiskEncryptionFormat::DISK_ENCRYPTION_FORMAT_LUKS1,
-          CipherAlgorithm::CIPHER_ALGORITHM_AES128, std::move(passphrase),
-          on_finish, true);
-  expect_crypto_layer_exists_check();
+          mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS1,
+          RBD_ENCRYPTION_ALGORITHM_AES128, std::move(passphrase), on_finish,
+          true);
   expect_get_object_size();
   expect_get_image_size(IMAGE_SIZE);
   expect_image_write();
@@ -126,15 +120,14 @@ TEST_F(TestMockCryptoLuksFormatRequest, LUKS1) {
   ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
   complete_aio(0);
   ASSERT_EQ(0, finished_cond.wait());
-  ASSERT_NO_FATAL_FAILURE(verify_header(32, 512));
+  ASSERT_NO_FATAL_FAILURE(verify_header(CRYPT_LUKS1, 32, 512));
 }
 
 TEST_F(TestMockCryptoLuksFormatRequest, AES128) {
   auto mock_format_request = MockFormatRequest::create(
-          mock_image_ctx, DiskEncryptionFormat::DISK_ENCRYPTION_FORMAT_LUKS2,
-          CipherAlgorithm::CIPHER_ALGORITHM_AES128, std::move(passphrase),
-          on_finish, true);
-  expect_crypto_layer_exists_check();
+          mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS2,
+          RBD_ENCRYPTION_ALGORITHM_AES128, std::move(passphrase), on_finish,
+          true);
   expect_get_object_size();
   expect_get_image_size(IMAGE_SIZE);
   expect_image_write();
@@ -142,15 +135,14 @@ TEST_F(TestMockCryptoLuksFormatRequest, AES128) {
   ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
   complete_aio(0);
   ASSERT_EQ(0, finished_cond.wait());
-  ASSERT_NO_FATAL_FAILURE(verify_header(32, 4096));
+  ASSERT_NO_FATAL_FAILURE(verify_header(CRYPT_LUKS2, 32, 4096));
 }
 
 TEST_F(TestMockCryptoLuksFormatRequest, AES256) {
   auto mock_format_request = MockFormatRequest::create(
-          mock_image_ctx, DiskEncryptionFormat::DISK_ENCRYPTION_FORMAT_LUKS2,
-          CipherAlgorithm::CIPHER_ALGORITHM_AES256, std::move(passphrase),
-          on_finish, true);
-  expect_crypto_layer_exists_check();
+          mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS2,
+          RBD_ENCRYPTION_ALGORITHM_AES256, std::move(passphrase), on_finish,
+          true);
   expect_get_object_size();
   expect_get_image_size(IMAGE_SIZE);
   expect_image_write();
@@ -158,17 +150,7 @@ TEST_F(TestMockCryptoLuksFormatRequest, AES256) {
   ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
   complete_aio(0);
   ASSERT_EQ(0, finished_cond.wait());
-  ASSERT_NO_FATAL_FAILURE(verify_header(62, 4096));
-}
-
-TEST_F(TestMockCryptoLuksFormatRequest, CryptoAlreadyLoaded) {
-  auto mock_format_request = MockFormatRequest::create(
-          mock_image_ctx, DiskEncryptionFormat::DISK_ENCRYPTION_FORMAT_LUKS2,
-          CipherAlgorithm::CIPHER_ALGORITHM_AES256, std::move(passphrase),
-          on_finish, true);
-  expect_crypto_layer_exists_check(true);
-  mock_format_request->send();
-  ASSERT_EQ(-EEXIST, finished_cond.wait());
+  ASSERT_NO_FATAL_FAILURE(verify_header(CRYPT_LUKS2, 62, 4096));
 }
 
 TEST_F(TestMockCryptoLuksFormatRequest, ImageTooSmall) {
@@ -184,10 +166,9 @@ TEST_F(TestMockCryptoLuksFormatRequest, ImageTooSmall) {
 
 TEST_F(TestMockCryptoLuksFormatRequest, WriteFail) {
   auto mock_format_request = MockFormatRequest::create(
-          mock_image_ctx, DiskEncryptionFormat::DISK_ENCRYPTION_FORMAT_LUKS2,
-          CipherAlgorithm::CIPHER_ALGORITHM_AES256, std::move(passphrase),
-          on_finish, true);
-  expect_crypto_layer_exists_check();
+          mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS2,
+          RBD_ENCRYPTION_ALGORITHM_AES256, std::move(passphrase), on_finish,
+          true);
   expect_get_object_size();
   expect_get_image_size(IMAGE_SIZE);
   expect_image_write();
index 5511fbe8f64513ed5aeaef867c419466c0c1a8bd..6f79f3c1f5855e1bb4a33fb3ed1399df6361f22b 100644 (file)
@@ -49,7 +49,8 @@ struct TestMockCryptoLuksLoadRequest : public TestMockFixture {
     mock_image_ctx = new MockImageCtx(*ictx);
     crypto = nullptr;
     mock_load_request = MockLoadRequest::create(
-            mock_image_ctx, std::move(passphrase), &crypto, on_finish);
+            mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS2, std::move(passphrase),
+            &crypto, on_finish);
   }
 
   void TearDown() override {
@@ -74,12 +75,7 @@ struct TestMockCryptoLuksLoadRequest : public TestMockFixture {
     data_offset = header.get_data_offset();
   }
 
-    void expect_crypto_layer_exists_check(bool exists = false) {
-      EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, exists(
-              io::OBJECT_DISPATCH_LAYER_CRYPTO)).WillOnce(Return(exists));
-    }
-
-    void expect_image_read(uint64_t offset, uint64_t length) {
+  void expect_image_read(uint64_t offset, uint64_t length) {
     EXPECT_CALL(*mock_image_ctx->io_image_dispatcher, send(_))
             .WillOnce(Invoke([this, offset,
                               length](io::ImageDispatchSpec* spec) {
@@ -109,7 +105,6 @@ struct TestMockCryptoLuksLoadRequest : public TestMockFixture {
 
 TEST_F(TestMockCryptoLuksLoadRequest, AES128) {
   generate_header(CRYPT_LUKS2, "aes", 32, "xts-plain64", 4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
   image_read_request->complete(DEFAULT_INITIAL_READ_SIZE);
@@ -119,7 +114,6 @@ TEST_F(TestMockCryptoLuksLoadRequest, AES128) {
 
 TEST_F(TestMockCryptoLuksLoadRequest, AES256) {
   generate_header(CRYPT_LUKS2, "aes", 64, "xts-plain64", 4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
   image_read_request->complete(DEFAULT_INITIAL_READ_SIZE);
@@ -128,8 +122,11 @@ TEST_F(TestMockCryptoLuksLoadRequest, AES256) {
 }
 
 TEST_F(TestMockCryptoLuksLoadRequest, LUKS1) {
+  delete mock_load_request;
+  mock_load_request = MockLoadRequest::create(
+          mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS1, {passphrase_cstr},
+          &crypto, on_finish);
   generate_header(CRYPT_LUKS1, "aes", 32, "xts-plain64", 512);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
   image_read_request->complete(DEFAULT_INITIAL_READ_SIZE);
@@ -137,9 +134,23 @@ TEST_F(TestMockCryptoLuksLoadRequest, LUKS1) {
   ASSERT_NE(crypto, nullptr);
 }
 
+TEST_F(TestMockCryptoLuksLoadRequest, WrongFormat) {
+  generate_header(CRYPT_LUKS1, "aes", 32, "xts-plain64", 512);
+  expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
+  mock_load_request->send();
+
+  expect_image_read(DEFAULT_INITIAL_READ_SIZE,
+                    MAXIMUM_HEADER_SIZE - DEFAULT_INITIAL_READ_SIZE);
+  image_read_request->complete(DEFAULT_INITIAL_READ_SIZE); // complete 1st read
+
+  image_read_request->complete(
+          MAXIMUM_HEADER_SIZE - DEFAULT_INITIAL_READ_SIZE);
+  ASSERT_EQ(-EINVAL, finished_cond.wait());
+  ASSERT_EQ(crypto, nullptr);
+}
+
 TEST_F(TestMockCryptoLuksLoadRequest, UnsupportedAlgorithm) {
   generate_header(CRYPT_LUKS2, "twofish", 32, "xts-plain64", 4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
   image_read_request->complete(DEFAULT_INITIAL_READ_SIZE);
@@ -149,7 +160,6 @@ TEST_F(TestMockCryptoLuksLoadRequest, UnsupportedAlgorithm) {
 
 TEST_F(TestMockCryptoLuksLoadRequest, UnsupportedCipherMode) {
   generate_header(CRYPT_LUKS2, "aes", 32, "cbc-essiv:sha256", 4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
   image_read_request->complete(DEFAULT_INITIAL_READ_SIZE);
@@ -160,7 +170,6 @@ TEST_F(TestMockCryptoLuksLoadRequest, UnsupportedCipherMode) {
 TEST_F(TestMockCryptoLuksLoadRequest, HeaderBiggerThanInitialRead) {
   generate_header(CRYPT_LUKS2, "aes", 64, "xts-plain64", 4096);
   mock_load_request->set_initial_read_size(4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, 4096);
   mock_load_request->send();
 
@@ -175,7 +184,6 @@ TEST_F(TestMockCryptoLuksLoadRequest, HeaderBiggerThanInitialRead) {
 TEST_F(TestMockCryptoLuksLoadRequest, KeyslotsBiggerThanInitialRead) {
   generate_header(CRYPT_LUKS2, "aes", 64, "xts-plain64", 4096);
   mock_load_request->set_initial_read_size(16384);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, 16384);
   mock_load_request->send();
 
@@ -190,10 +198,10 @@ TEST_F(TestMockCryptoLuksLoadRequest, KeyslotsBiggerThanInitialRead) {
 TEST_F(TestMockCryptoLuksLoadRequest, WrongPassphrase) {
   delete mock_load_request;
   mock_load_request = MockLoadRequest::create(
-        mock_image_ctx, "wrong", &crypto, on_finish);
+        mock_image_ctx, RBD_ENCRYPTION_FORMAT_LUKS2, "wrong", &crypto,
+        on_finish);
 
   generate_header(CRYPT_LUKS2, "aes", 64, "xts-plain64", 4096);
-  expect_crypto_layer_exists_check();
   expect_image_read(0, DEFAULT_INITIAL_READ_SIZE);
   mock_load_request->send();
 
@@ -207,14 +215,6 @@ TEST_F(TestMockCryptoLuksLoadRequest, WrongPassphrase) {
   ASSERT_EQ(crypto, nullptr);
 }
 
-TEST_F(TestMockCryptoLuksLoadRequest, CryptoAlreadyLoaded) {
-  generate_header(CRYPT_LUKS2, "aes", 32, "xts-plain64", 4096);
-  expect_crypto_layer_exists_check(true);
-  mock_load_request->send();
-  ASSERT_EQ(-EEXIST, finished_cond.wait());
-  ASSERT_EQ(crypto, nullptr);
-}
-
 } // namespace luks
 } // namespace crypto
 } // namespace librbd
diff --git a/src/test/librbd/crypto/test_mock_FormatRequest.cc b/src/test/librbd/crypto/test_mock_FormatRequest.cc
new file mode 100644 (file)
index 0000000..722f768
--- /dev/null
@@ -0,0 +1,122 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_mock_fixture.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librbd/mock/crypto/MockEncryptionFormat.h"
+
+#include "librbd/crypto/FormatRequest.cc"
+
+namespace librbd {
+namespace crypto {
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::WithArg;
+
+struct TestMockCryptoFormatRequest : public TestMockFixture {
+  typedef FormatRequest<librbd::MockImageCtx> MockFormatRequest;
+
+  MockImageCtx* mock_image_ctx;
+  C_SaferCond finished_cond;
+  Context *on_finish = &finished_cond;
+  MockEncryptionFormat* mock_encryption_format;
+  Context* format_context;
+
+  void SetUp() override {
+    TestMockFixture::SetUp();
+
+    librbd::ImageCtx *ictx;
+    ASSERT_EQ(0, open_image(m_image_name, &ictx));
+    mock_image_ctx = new MockImageCtx(*ictx);
+    mock_encryption_format = new MockEncryptionFormat();
+  }
+
+  void TearDown() override {
+    delete mock_image_ctx;
+    TestMockFixture::TearDown();
+  }
+
+  void expect_crypto_layer_exists_check(bool exists) {
+    EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, exists(
+            io::OBJECT_DISPATCH_LAYER_CRYPTO)).WillOnce(Return(exists));
+  }
+
+  void expect_test_journal_feature(bool has_journal=false) {
+    EXPECT_CALL(*mock_image_ctx, test_features(
+            RBD_FEATURE_JOURNALING)).WillOnce(Return(has_journal));
+  }
+
+  void expect_encryption_format() {
+    EXPECT_CALL(*mock_encryption_format, format(
+            mock_image_ctx, _)).WillOnce(
+                    WithArg<1>(Invoke([this](Context* ctx) {
+                      format_context = ctx;
+    })));
+  }
+};
+
+TEST_F(TestMockCryptoFormatRequest, CryptoAlreadyLoaded) {
+  auto mock_format_request = MockFormatRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(true);
+  mock_format_request->send();
+  ASSERT_EQ(-EEXIST, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoFormatRequest, JournalEnabled) {
+  auto mock_format_request = MockFormatRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(true);
+  mock_format_request->send();
+  ASSERT_EQ(-ENOTSUP, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoFormatRequest, CloneFormat) {
+  mock_image_ctx->parent = mock_image_ctx;
+  auto mock_format_request = MockFormatRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  mock_format_request->send();
+  ASSERT_EQ(-ENOTSUP, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoFormatRequest, FormatFail) {
+  auto mock_format_request = MockFormatRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(false);
+  expect_encryption_format();
+  mock_format_request->send();
+  ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
+  format_context->complete(-EIO);
+  ASSERT_EQ(-EIO, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoFormatRequest, Success) {
+  auto mock_format_request = MockFormatRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(false);
+  expect_encryption_format();
+  mock_format_request->send();
+  ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
+  format_context->complete(0);
+  ASSERT_EQ(0, finished_cond.wait());
+}
+
+} // namespace crypto
+} // namespace librbd
diff --git a/src/test/librbd/crypto/test_mock_LoadRequest.cc b/src/test/librbd/crypto/test_mock_LoadRequest.cc
new file mode 100644 (file)
index 0000000..ac36b65
--- /dev/null
@@ -0,0 +1,137 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/crypto/CryptoObjectDispatch.h"
+#include "test/librbd/test_mock_fixture.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librbd/mock/crypto/MockCryptoInterface.h"
+#include "test/librbd/mock/crypto/MockEncryptionFormat.h"
+#include "test/librbd/mock/io/MockObjectDispatch.h"
+
+namespace librbd {
+namespace crypto {
+
+template <>
+struct CryptoObjectDispatch<MockImageCtx> : public io::MockObjectDispatch {
+
+  static CryptoObjectDispatch* create(
+          MockImageCtx* image_ctx,ceph::ref_t<CryptoInterface> crypto) {
+    return new CryptoObjectDispatch();
+  }
+
+  CryptoObjectDispatch() {
+  }
+};
+
+} // namespace crypto
+} // namespace librbd
+
+#include "librbd/crypto/LoadRequest.cc"
+
+namespace librbd {
+namespace crypto {
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::WithArgs;
+
+struct TestMockCryptoLoadRequest : public TestMockFixture {
+  typedef LoadRequest<librbd::MockImageCtx> MockLoadRequest;
+
+  MockImageCtx* mock_image_ctx;
+  C_SaferCond finished_cond;
+  Context *on_finish = &finished_cond;
+  MockEncryptionFormat* mock_encryption_format;
+  Context* load_context;
+
+  void SetUp() override {
+    TestMockFixture::SetUp();
+
+    librbd::ImageCtx *ictx;
+    ASSERT_EQ(0, open_image(m_image_name, &ictx));
+    mock_image_ctx = new MockImageCtx(*ictx);
+    mock_encryption_format = new MockEncryptionFormat();
+  }
+
+  void TearDown() override {
+    delete mock_image_ctx;
+    TestMockFixture::TearDown();
+  }
+
+  void expect_crypto_layer_exists_check(bool exists) {
+    EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, exists(
+            io::OBJECT_DISPATCH_LAYER_CRYPTO)).WillOnce(Return(exists));
+  }
+
+  void expect_test_journal_feature(bool has_journal=false) {
+    EXPECT_CALL(*mock_image_ctx, test_features(
+            RBD_FEATURE_JOURNALING)).WillOnce(Return(has_journal));
+  }
+
+  void expect_encryption_load() {
+    EXPECT_CALL(*mock_encryption_format, load(
+            mock_image_ctx, _, _)).WillOnce(
+                    WithArgs<1, 2>(Invoke([this](
+                            ceph::ref_t<CryptoInterface>* result_crypto,
+                            Context* ctx) {
+                      load_context = ctx;
+                      *result_crypto = new MockCryptoInterface();
+    })));
+  }
+};
+
+TEST_F(TestMockCryptoLoadRequest, CryptoAlreadyLoaded) {
+  auto mock_load_request = MockLoadRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(true);
+  mock_load_request->send();
+  ASSERT_EQ(-EEXIST, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoLoadRequest, JournalEnabled) {
+  auto mock_load_request = MockLoadRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(true);
+  mock_load_request->send();
+  ASSERT_EQ(-ENOTSUP, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoLoadRequest, LoadFail) {
+  auto mock_load_request = MockLoadRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(false);
+  expect_encryption_load();
+  mock_load_request->send();
+  ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
+  load_context->complete(-EIO);
+  ASSERT_EQ(-EIO, finished_cond.wait());
+}
+
+TEST_F(TestMockCryptoLoadRequest, Success) {
+  auto mock_load_request = MockLoadRequest::create(
+          mock_image_ctx,
+          std::unique_ptr<MockEncryptionFormat>(mock_encryption_format),
+          on_finish);
+  expect_crypto_layer_exists_check(false);
+  expect_test_journal_feature(false);
+  expect_encryption_load();
+  mock_load_request->send();
+  ASSERT_EQ(ETIMEDOUT, finished_cond.wait_for(0));
+  EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, register_dispatch(_));
+  EXPECT_CALL(*mock_image_ctx->io_image_dispatcher, register_dispatch(_));
+  load_context->complete(0);
+  ASSERT_EQ(0, finished_cond.wait());
+}
+
+} // namespace crypto
+} // namespace librbd
index 429a58410e986ec1cf5757f42b87af3c9bdfa417..dcbd7c21a1265ad0dd4acc95cf44e5110838dc8f 100644 (file)
@@ -154,6 +154,7 @@ struct MockImageCtx {
   MOCK_CONST_METHOD0(get_object_size, uint64_t());
   MOCK_CONST_METHOD0(get_current_size, uint64_t());
   MOCK_CONST_METHOD1(get_image_size, uint64_t(librados::snap_t));
+  MOCK_CONST_METHOD1(get_effective_image_size, uint64_t(librados::snap_t));
   MOCK_CONST_METHOD1(get_object_count, uint64_t(librados::snap_t));
   MOCK_CONST_METHOD1(get_read_flags, int(librados::snap_t));
   MOCK_CONST_METHOD2(get_flags, int(librados::snap_t in_snap_id,
diff --git a/src/test/librbd/mock/crypto/MockEncryptionFormat.h b/src/test/librbd/mock/crypto/MockEncryptionFormat.h
new file mode 100644 (file)
index 0000000..751ee82
--- /dev/null
@@ -0,0 +1,42 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_LIBRBD_MOCK_CRYPTO_MOCK_ENCRYPTION_FORMAT_H
+#define CEPH_TEST_LIBRBD_MOCK_CRYPTO_MOCK_ENCRYPTION_FORMAT_H
+
+#include "gmock/gmock.h"
+#include "librbd/crypto/EncryptionFormat.h"
+#include "test/librbd/mock/MockImageCtx.h"
+
+namespace librbd {
+namespace crypto {
+
+struct MockEncryptionFormat : EncryptionFormat<MockImageCtx> {
+
+  MOCK_METHOD2(format, void(MockImageCtx* ictx, Context* on_finish));
+  MOCK_METHOD3(load, void(MockImageCtx* ictx,
+                          ceph::ref_t<CryptoInterface>* result_crypto,
+                          Context* on_finish));
+};
+
+} // namespace crypto
+
+namespace api {
+namespace util {
+
+inline int create_encryption_format(
+        CephContext* cct, encryption_format_t format,
+        encryption_options_t opts, size_t opts_size, bool c_api,
+        crypto::EncryptionFormat<MockImageCtx>** result_format) {
+  if (opts == nullptr) {
+    return -ENOTSUP;
+  }
+  *result_format = (crypto::MockEncryptionFormat*)opts;
+  return 0;
+}
+
+} // namespace util
+} // namespace api
+} // namespace librbd
+
+#endif // CEPH_TEST_LIBRBD_MOCK_CRYPTO_MOCK_ENCRYPTION_FORMAT_H
index 0330a0e2810baab8c2f06377090ca76f3df22c7f..f64e6f435ea524e26c020f44fa1af1d639ebd989 100644 (file)
@@ -179,6 +179,314 @@ static int create_image_pp(librbd::RBD &rbd,
   }
 }
 
+
+
+void simple_write_cb(rbd_completion_t cb, void *arg)
+{
+  printf("write completion cb called!\n");
+}
+
+void simple_read_cb(rbd_completion_t cb, void *arg)
+{
+  printf("read completion cb called!\n");
+}
+
+void aio_write_test_data_and_poll(rbd_image_t image, int fd, const char *test_data,
+                                  uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  uint64_t data = 0x123;
+  rbd_aio_create_completion((void*)&data, (rbd_callback_t) simple_write_cb, &comp);
+  printf("created completion\n");
+  printf("started write\n");
+  if (iohint)
+    rbd_aio_write2(image, off, len, test_data, comp, iohint);
+  else
+    rbd_aio_write(image, off, len, test_data, comp);
+
+  struct pollfd pfd;
+  pfd.fd = fd;
+  pfd.events = POLLIN;
+
+  ASSERT_EQ(1, poll(&pfd, 1, -1));
+  ASSERT_TRUE(pfd.revents & POLLIN);
+
+  rbd_completion_t comps[1];
+  ASSERT_EQ(1, rbd_poll_io_events(image, comps, 1));
+  uint64_t count;
+  ASSERT_EQ(static_cast<ssize_t>(sizeof(count)),
+            read(fd, &count, sizeof(count)));
+  int r = rbd_aio_get_return_value(comps[0]);
+  ASSERT_TRUE(rbd_aio_is_complete(comps[0]));
+  ASSERT_TRUE(*(uint64_t*)rbd_aio_get_arg(comps[0]) == data);
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(0, r);
+  printf("finished write\n");
+  rbd_aio_release(comps[0]);
+  *passed = true;
+}
+
+void aio_write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  printf("created completion\n");
+  if (iohint)
+    rbd_aio_write2(image, off, len, test_data, comp, iohint);
+  else
+    rbd_aio_write(image, off, len, test_data, comp);
+  printf("started write\n");
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(0, r);
+  printf("finished write\n");
+  rbd_aio_release(comp);
+  *passed = true;
+}
+
+void write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  ssize_t written;
+  if (iohint)
+    written = rbd_write2(image, off, len, test_data, iohint);
+  else
+    written = rbd_write(image, off, len, test_data);
+  printf("wrote: %d\n", (int) written);
+  ASSERT_EQ(len, static_cast<size_t>(written));
+  *passed = true;
+}
+
+void aio_discard_test_data(rbd_image_t image, uint64_t off, uint64_t len, bool *passed)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  rbd_aio_discard(image, off, len, comp);
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  ASSERT_EQ(0, r);
+  printf("aio discard: %d~%d = %d\n", (int)off, (int)len, (int)r);
+  rbd_aio_release(comp);
+  *passed = true;
+}
+
+void discard_test_data(rbd_image_t image, uint64_t off, size_t len, bool *passed)
+{
+  ssize_t written;
+  written = rbd_discard(image, off, len);
+  printf("discard: %d~%d = %d\n", (int)off, (int)len, (int)written);
+  ASSERT_EQ(len, static_cast<size_t>(written));
+  *passed = true;
+}
+
+void aio_read_test_data_and_poll(rbd_image_t image, int fd, const char *expected,
+                                 uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  char *result = (char *)malloc(len + 1);
+
+  ASSERT_NE(static_cast<char *>(NULL), result);
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+  printf("created completion\n");
+  printf("started read\n");
+  if (iohint)
+    rbd_aio_read2(image, off, len, result, comp, iohint);
+  else
+    rbd_aio_read(image, off, len, result, comp);
+
+  struct pollfd pfd;
+  pfd.fd = fd;
+  pfd.events = POLLIN;
+
+  ASSERT_EQ(1, poll(&pfd, 1, -1));
+  ASSERT_TRUE(pfd.revents & POLLIN);
+
+  rbd_completion_t comps[1];
+  ASSERT_EQ(1, rbd_poll_io_events(image, comps, 1));
+  uint64_t count;
+  ASSERT_EQ(static_cast<ssize_t>(sizeof(count)),
+            read(fd, &count, sizeof(count)));
+
+  int r = rbd_aio_get_return_value(comps[0]);
+  ASSERT_TRUE(rbd_aio_is_complete(comps[0]));
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(len, static_cast<size_t>(r));
+  rbd_aio_release(comps[0]);
+  if (memcmp(result, expected, len)) {
+    printf("read: %s\nexpected: %s\n", result, expected);
+    ASSERT_EQ(0, memcmp(result, expected, len));
+  }
+  free(result);
+  *passed = true;
+}
+
+void aio_read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  char *result = (char *)malloc(len + 1);
+
+  ASSERT_NE(static_cast<char *>(NULL), result);
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+  printf("created completion\n");
+  if (iohint)
+    rbd_aio_read2(image, off, len, result, comp, iohint);
+  else
+    rbd_aio_read(image, off, len, result, comp);
+  printf("started read\n");
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(len, static_cast<size_t>(r));
+  rbd_aio_release(comp);
+  if (memcmp(result, expected, len)) {
+    printf("read: %s\nexpected: %s\n", result, expected);
+    ASSERT_EQ(0, memcmp(result, expected, len));
+  }
+  free(result);
+  *passed = true;
+}
+
+void read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+{
+  ssize_t read;
+  char *result = (char *)malloc(len + 1);
+
+  ASSERT_NE(static_cast<char *>(NULL), result);
+  if (iohint)
+    read = rbd_read2(image, off, len, result, iohint);
+  else
+    read = rbd_read(image, off, len, result);
+  printf("read: %d\n", (int) read);
+  ASSERT_EQ(len, static_cast<size_t>(read));
+  result[len] = '\0';
+  if (memcmp(result, expected, len)) {
+    printf("read: %s\nexpected: %s\n", result, expected);
+    ASSERT_EQ(0, memcmp(result, expected, len));
+  }
+  free(result);
+  *passed = true;
+}
+
+void aio_writesame_test_data(rbd_image_t image, const char *test_data, uint64_t off, uint64_t len,
+                             uint64_t data_len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  printf("created completion\n");
+  int r;
+  r = rbd_aio_writesame(image, off, len, test_data, data_len, comp, iohint);
+  printf("started writesame\n");
+  if (len % data_len) {
+    ASSERT_EQ(-EINVAL, r);
+    printf("expected fail, finished writesame\n");
+    rbd_aio_release(comp);
+    *passed = true;
+    return;
+  }
+
+  rbd_aio_wait_for_complete(comp);
+  r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(0, r);
+  printf("finished writesame\n");
+  rbd_aio_release(comp);
+
+  //verify data
+  printf("to verify the data\n");
+  ssize_t read;
+  char *result = (char *)malloc(data_len+ 1);
+  ASSERT_NE(static_cast<char *>(NULL), result);
+  uint64_t left = len;
+  while (left > 0) {
+    read = rbd_read(image, off, data_len, result);
+    ASSERT_EQ(data_len, static_cast<size_t>(read));
+    result[data_len] = '\0';
+    if (memcmp(result, test_data, data_len)) {
+      printf("read: %d ~ %d\n", (int) off, (int) read);
+      printf("read: %s\nexpected: %s\n", result, test_data);
+      ASSERT_EQ(0, memcmp(result, test_data, data_len));
+    }
+    off += data_len;
+    left -= data_len;
+  }
+  ASSERT_EQ(0U, left);
+  free(result);
+  printf("verified\n");
+
+  *passed = true;
+}
+
+void writesame_test_data(rbd_image_t image, const char *test_data, uint64_t off, uint64_t len,
+                         uint64_t data_len, uint32_t iohint, bool *passed)
+{
+  ssize_t written;
+  written = rbd_writesame(image, off, len, test_data, data_len, iohint);
+  if (len % data_len) {
+    ASSERT_EQ(-EINVAL, written);
+    printf("expected fail, finished writesame\n");
+    *passed = true;
+    return;
+  }
+  ASSERT_EQ(len, static_cast<size_t>(written));
+  printf("wrote: %d\n", (int) written);
+
+  //verify data
+  printf("to verify the data\n");
+  ssize_t read;
+  char *result = (char *)malloc(data_len+ 1);
+  ASSERT_NE(static_cast<char *>(NULL), result);
+  uint64_t left = len;
+  while (left > 0) {
+    read = rbd_read(image, off, data_len, result);
+    ASSERT_EQ(data_len, static_cast<size_t>(read));
+    result[data_len] = '\0';
+    if (memcmp(result, test_data, data_len)) {
+      printf("read: %d ~ %d\n", (int) off, (int) read);
+      printf("read: %s\nexpected: %s\n", result, test_data);
+      ASSERT_EQ(0, memcmp(result, test_data, data_len));
+    }
+    off += data_len;
+    left -= data_len;
+  }
+  ASSERT_EQ(0U, left);
+  free(result);
+  printf("verified\n");
+
+  *passed = true;
+}
+
+void aio_compare_and_write_test_data(rbd_image_t image, const char *cmp_data,
+                                     const char *test_data, uint64_t off,
+                                     size_t len, uint32_t iohint, bool *passed)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  printf("created completion\n");
+
+  uint64_t mismatch_offset;
+  rbd_aio_compare_and_write(image, off, len, cmp_data, test_data, comp, &mismatch_offset, iohint);
+  printf("started aio compare and write\n");
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  ASSERT_EQ(0, r);
+  printf("finished aio compare and write\n");
+  rbd_aio_release(comp);
+  *passed = true;
+}
+
+void compare_and_write_test_data(rbd_image_t image, const char *cmp_data,
+                                 const char *test_data, uint64_t off, size_t len,
+                                 uint64_t *mismatch_off, uint32_t iohint, bool *passed)
+{
+  printf("start compare and write\n");
+  ssize_t written;
+  written = rbd_compare_and_write(image, off, len, cmp_data, test_data, mismatch_off, iohint);
+  printf("compare and  wrote: %d\n", (int) written);
+  ASSERT_EQ(len, static_cast<size_t>(written));
+  *passed = true;
+}
+
 class TestLibRBD : public ::testing::Test {
 public:
 
@@ -281,8 +589,139 @@ public:
       EXPECT_EQ("", create_one_pool_pp(pool_name, rados));
       _pool_names.push_back(pool_name);
     }
-    ++m_pool_number;
-    return pool_name;
+    ++m_pool_number;
+    return pool_name;
+  }
+
+  void test_io(rbd_image_t image) {
+    bool skip_discard = is_skip_partial_discard_enabled(image);
+
+    char test_data[TEST_IO_SIZE + 1];
+    char zero_data[TEST_IO_SIZE + 1];
+    char mismatch_data[TEST_IO_SIZE + 1];
+    int i;
+    uint64_t mismatch_offset;
+
+    for (i = 0; i < TEST_IO_SIZE; ++i) {
+      test_data[i] = (char) (rand() % (126 - 33) + 33);
+    }
+    test_data[TEST_IO_SIZE] = '\0';
+    memset(zero_data, 0, sizeof(zero_data));
+    memset(mismatch_data, 9, sizeof(mismatch_data));
+
+    for (i = 0; i < 5; ++i)
+      ASSERT_PASSED(write_test_data, image, test_data, TEST_IO_SIZE * i,
+                    TEST_IO_SIZE, 0);
+
+    for (i = 5; i < 10; ++i)
+      ASSERT_PASSED(aio_write_test_data, image, test_data, TEST_IO_SIZE * i,
+                    TEST_IO_SIZE, 0);
+
+    for (i = 0; i < 5; ++i)
+      ASSERT_PASSED(compare_and_write_test_data, image, test_data, test_data,
+                    TEST_IO_SIZE * i, TEST_IO_SIZE, &mismatch_offset, 0);
+
+    for (i = 5; i < 10; ++i)
+      ASSERT_PASSED(aio_compare_and_write_test_data, image, test_data, test_data,
+                    TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+
+    for (i = 0; i < 5; ++i)
+      ASSERT_PASSED(read_test_data, image, test_data, TEST_IO_SIZE * i,
+                    TEST_IO_SIZE, 0);
+
+    for (i = 5; i < 10; ++i)
+      ASSERT_PASSED(aio_read_test_data, image, test_data, TEST_IO_SIZE * i,
+                    TEST_IO_SIZE, 0);
+
+    // discard 2nd, 4th sections.
+    ASSERT_PASSED(discard_test_data, image, TEST_IO_SIZE, TEST_IO_SIZE);
+    ASSERT_PASSED(aio_discard_test_data, image, TEST_IO_SIZE*3, TEST_IO_SIZE);
+
+    ASSERT_PASSED(read_test_data, image, test_data,  0, TEST_IO_SIZE, 0);
+    ASSERT_PASSED(read_test_data, image, skip_discard ? test_data : zero_data,
+                  TEST_IO_SIZE, TEST_IO_SIZE, 0);
+    ASSERT_PASSED(read_test_data, image, test_data,  TEST_IO_SIZE*2,
+                  TEST_IO_SIZE, 0);
+    ASSERT_PASSED(read_test_data, image, skip_discard ? test_data : zero_data,
+                  TEST_IO_SIZE*3, TEST_IO_SIZE, 0);
+    ASSERT_PASSED(read_test_data, image, test_data,  TEST_IO_SIZE*4,
+                  TEST_IO_SIZE, 0);
+
+    for (i = 0; i < 15; ++i) {
+      if (i % 3 == 2) {
+        ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE * i,
+                      TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
+        ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE * i,
+                      TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
+      } else if (i % 3 == 1) {
+        ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE + i,
+                      TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+        ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE + i,
+                      TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+      } else {
+        ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE * i,
+                      TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+        ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE * i,
+                      TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+      }
+    }
+    for (i = 0; i < 15; ++i) {
+      if (i % 3 == 2) {
+        ASSERT_PASSED(aio_writesame_test_data, image, test_data,
+                      TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE,
+                      0);
+        ASSERT_PASSED(aio_writesame_test_data, image, zero_data,
+                      TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE,
+                      0);
+      } else if (i % 3 == 1) {
+        ASSERT_PASSED(aio_writesame_test_data, image, test_data,
+                      TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+        ASSERT_PASSED(aio_writesame_test_data, image, zero_data,
+                      TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+      } else {
+        ASSERT_PASSED(aio_writesame_test_data, image, test_data,
+                      TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+        ASSERT_PASSED(aio_writesame_test_data, image, zero_data,
+                      TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
+      }
+    }
+
+    rbd_image_info_t info;
+    rbd_completion_t comp;
+    ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
+    // can't read or write starting past end
+    ASSERT_EQ(-EINVAL, rbd_write(image, info.size, 1, test_data));
+    ASSERT_EQ(-EINVAL, rbd_read(image, info.size, 1, test_data));
+    // reading through end returns amount up to end
+    ASSERT_EQ(10, rbd_read(image, info.size - 10, 100, test_data));
+    // writing through end returns amount up to end
+    ASSERT_EQ(10, rbd_write(image, info.size - 10, 100, test_data));
+
+    rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+    ASSERT_EQ(0, rbd_aio_write(image, info.size, 1, test_data, comp));
+    ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
+    ASSERT_EQ(-EINVAL, rbd_aio_get_return_value(comp));
+    rbd_aio_release(comp);
+
+    rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+    ASSERT_EQ(0, rbd_aio_read(image, info.size, 1, test_data, comp));
+    ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
+    ASSERT_EQ(-EINVAL, rbd_aio_get_return_value(comp));
+    rbd_aio_release(comp);
+
+    ASSERT_PASSED(write_test_data, image, zero_data, 0, TEST_IO_SIZE,
+                  LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+    ASSERT_EQ(-EILSEQ, rbd_compare_and_write(image, 0, TEST_IO_SIZE,
+              mismatch_data, mismatch_data, &mismatch_offset, 0));
+    ASSERT_EQ(0U, mismatch_offset);
+    rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+    ASSERT_EQ(0, rbd_aio_compare_and_write(image, 0, TEST_IO_SIZE, mismatch_data,
+              mismatch_data, comp, &mismatch_offset, 0));
+    ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
+    ASSERT_EQ(0U, mismatch_offset);
+    rbd_aio_release(comp);
+
+    ASSERT_PASSED(validate_object_map, image);
   }
 
   static std::vector<std::string> _pool_names;
@@ -1270,693 +1709,386 @@ TEST_F(TestLibRBD, TestDeepCopy)
 
   keys_len = sizeof(keys);
   vals_len = sizeof(vals);
-  ASSERT_EQ(0, rbd_open(ioctx, name6.c_str(), &image6, NULL));
-  BOOST_SCOPE_EXIT_ALL( (&image6) ) {
-    ASSERT_EQ(0, rbd_close(image6));
-  };
-  ASSERT_EQ(0, rbd_metadata_list(image6, "key", 70, keys, &keys_len, vals,
-                                 &vals_len));
-  ASSERT_EQ(keys_len, sum_key_len);
-  ASSERT_EQ(vals_len, sum_value_len);
-
-  for (int i = 1; i <= 70; i++) {
-    key = "key" + stringify(i);
-    val = "value" + stringify(i);
-    ASSERT_EQ(0, rbd_metadata_get(image6, key.c_str(), value, &value_len));
-    ASSERT_STREQ(val.c_str(), value);
-
-    value_len = sizeof(value);
-  }
-}
-
-TEST_F(TestLibRBD, TestDeepCopyPP)
-{
-  REQUIRE_FORMAT_V2();
-
-  librados::IoCtx ioctx;
-  ASSERT_EQ(0, _rados.ioctx_create(create_pool(true).c_str(), ioctx));
-
-  {
-    librbd::RBD rbd;
-    librbd::Image image;
-    librbd::Image image2;
-    librbd::Image image3;
-    int order = 0;
-    std::string name = get_temp_image_name();
-    std::string name2 = get_temp_image_name();
-    std::string name3 = get_temp_image_name();
-    uint64_t size = 2 << 20;
-    librbd::ImageOptions opts;
-    PrintProgress pp;
-
-    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
-    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
-
-    std::string key;
-    std::string val;
-    for (int i = 1; i <= 70; i++) {
-      key = "key" + stringify(i);
-      val = "value" + stringify(i);
-      ASSERT_EQ(0, image.metadata_set(key, val));
-    }
-
-    ASSERT_EQ(1, test_ls_pp(rbd, ioctx, 1, name.c_str()));
-    ASSERT_EQ(0, image.deep_copy(ioctx, name2.c_str(), opts));
-    ASSERT_EQ(2, test_ls_pp(rbd, ioctx, 2, name.c_str(), name2.c_str()));
-    ASSERT_EQ(0, rbd.open(ioctx, image2, name2.c_str(), NULL));
-
-    map<string, bufferlist> pairs;
-    std::string value;
-    ASSERT_EQ(0, image2.metadata_list("", 70, &pairs));
-    ASSERT_EQ(70U, pairs.size());
-
-    for (int i = 1; i <= 70; i++) {
-      key = "key" + stringify(i);
-      val = "value" + stringify(i);
-      ASSERT_EQ(0, image2.metadata_get(key.c_str(), &value));
-      ASSERT_STREQ(val.c_str(), value.c_str());
-    }
-
-    ASSERT_EQ(0, image.deep_copy_with_progress(ioctx, name3.c_str(), opts, pp));
-    ASSERT_EQ(3, test_ls_pp(rbd, ioctx, 3, name.c_str(), name2.c_str(),
-                            name3.c_str()));
-    ASSERT_EQ(0, rbd.open(ioctx, image3, name3.c_str(), NULL));
-
-    pairs.clear();
-    ASSERT_EQ(0, image3.metadata_list("", 70, &pairs));
-    ASSERT_EQ(70U, pairs.size());
-
-    for (int i = 1; i <= 70; i++) {
-      key = "key" + stringify(i);
-      val = "value" + stringify(i);
-      ASSERT_EQ(0, image3.metadata_get(key.c_str(), &value));
-      ASSERT_STREQ(val.c_str(), value.c_str());
-    }
-  }
-
-  ioctx.close();
-}
-
-int test_ls_snaps(rbd_image_t image, int num_expected, ...)
-{
-  int num_snaps, i, j, max_size = 10;
-  va_list ap;
-  rbd_snap_info_t snaps[max_size];
-  num_snaps = rbd_snap_list(image, snaps, &max_size);
-  printf("num snaps is: %d\nexpected: %d\n", num_snaps, num_expected);
-
-  for (i = 0; i < num_snaps; i++) {
-    printf("snap: %s\n", snaps[i].name);
-  }
-
-  va_start(ap, num_expected);
-  for (i = num_expected; i > 0; i--) {
-    char *expected = va_arg(ap, char *);
-    uint64_t expected_size = va_arg(ap, uint64_t);
-    bool found = false;
-    for (j = 0; j < num_snaps; j++) {
-      if (snaps[j].name == NULL)
-       continue;
-      if (strcmp(snaps[j].name, expected) == 0) {
-       printf("found %s with size %llu\n", snaps[j].name, (unsigned long long) snaps[j].size);
-       EXPECT_EQ(expected_size, snaps[j].size);
-       free((void *) snaps[j].name);
-       snaps[j].name = NULL;
-       found = true;
-       break;
-      }
-    }
-    EXPECT_TRUE(found);
-  }
-  va_end(ap);
-
-  for (i = 0; i < num_snaps; i++) {
-    EXPECT_EQ((const char *)0, snaps[i].name);
-  }
-
-  return num_snaps;
-}
-
-TEST_F(TestLibRBD, TestCreateLsDeleteSnap)
-{
-  rados_ioctx_t ioctx;
-  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
-
-  rbd_image_t image;
-  int order = 0;
-  std::string name = get_temp_image_name();
-  uint64_t size = 2 << 20;
-  uint64_t size2 = 4 << 20;
-  
-  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
-  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
-
-  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
-  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
-  ASSERT_EQ(0, rbd_resize(image, size2));
-  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
-  ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
-  ASSERT_EQ(0, rbd_snap_remove(image, "snap1"));
-  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
-  ASSERT_EQ(0, rbd_snap_remove(image, "snap2"));
-  ASSERT_EQ(0, test_ls_snaps(image, 0));
-  
-  ASSERT_EQ(0, rbd_close(image));
-
-  rados_ioctx_destroy(ioctx);
-}
-
-int test_get_snapshot_timestamp(rbd_image_t image, uint64_t snap_id)
-{
-  struct timespec timestamp;
-  EXPECT_EQ(0, rbd_snap_get_timestamp(image, snap_id, &timestamp));
-  EXPECT_LT(0, timestamp.tv_sec);
-  return 0;
-}
-
-TEST_F(TestLibRBD, TestGetSnapShotTimeStamp)
-{
-  REQUIRE_FORMAT_V2();
-
-  rados_ioctx_t ioctx;
-  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
-
-  rbd_image_t image;
-  int order = 0;
-  std::string name = get_temp_image_name();
-  uint64_t size = 2 << 20;
-  int num_snaps, max_size = 10;
-  rbd_snap_info_t snaps[max_size];
-
-  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
-  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
-
-  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
-  num_snaps = rbd_snap_list(image, snaps, &max_size);
-  ASSERT_EQ(1, num_snaps);
-  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[0].id));
-  free((void *)snaps[0].name);
-
-  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
-  num_snaps = rbd_snap_list(image, snaps, &max_size);
-  ASSERT_EQ(2, num_snaps);
-  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[0].id));
-  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[1].id));
-  free((void *)snaps[0].name);
-  free((void *)snaps[1].name);
-
-  ASSERT_EQ(0, rbd_close(image));
-
-  rados_ioctx_destroy(ioctx);
-}
-
-
-int test_ls_snaps(librbd::Image& image, size_t num_expected, ...)
-{
-  int r;
-  size_t i, j;
-  va_list ap;
-  vector<librbd::snap_info_t> snaps;
-  r = image.snap_list(snaps);
-  EXPECT_TRUE(r >= 0);
-  cout << "num snaps is: " << snaps.size() << std::endl
-           << "expected: " << num_expected << std::endl;
-
-  for (i = 0; i < snaps.size(); i++) {
-    cout << "snap: " << snaps[i].name << std::endl;
-  }
-
-  va_start(ap, num_expected);
-  for (i = num_expected; i > 0; i--) {
-    char *expected = va_arg(ap, char *);
-    uint64_t expected_size = va_arg(ap, uint64_t);
-    int found = 0;
-    for (j = 0; j < snaps.size(); j++) {
-      if (snaps[j].name == "")
-       continue;
-      if (strcmp(snaps[j].name.c_str(), expected) == 0) {
-       cout << "found " << snaps[j].name << " with size " << snaps[j].size
-            << std::endl;
-       EXPECT_EQ(expected_size, snaps[j].size);
-       snaps[j].name = "";
-       found = 1;
-       break;
-      }
-    }
-    EXPECT_TRUE(found);
-  }
-  va_end(ap);
-
-  for (i = 0; i < snaps.size(); i++) {
-    EXPECT_EQ("", snaps[i].name);
-  }
-
-  return snaps.size();
-}
+  ASSERT_EQ(0, rbd_open(ioctx, name6.c_str(), &image6, NULL));
+  BOOST_SCOPE_EXIT_ALL( (&image6) ) {
+    ASSERT_EQ(0, rbd_close(image6));
+  };
+  ASSERT_EQ(0, rbd_metadata_list(image6, "key", 70, keys, &keys_len, vals,
+                                 &vals_len));
+  ASSERT_EQ(keys_len, sum_key_len);
+  ASSERT_EQ(vals_len, sum_value_len);
 
-TEST_F(TestLibRBD, TestCreateLsDeleteSnapPP)
-{
-  librados::IoCtx ioctx;
-  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
+  for (int i = 1; i <= 70; i++) {
+    key = "key" + stringify(i);
+    val = "value" + stringify(i);
+    ASSERT_EQ(0, rbd_metadata_get(image6, key.c_str(), value, &value_len));
+    ASSERT_STREQ(val.c_str(), value);
 
-  {
-    librbd::RBD rbd;
-    librbd::Image image;
-    int order = 0;
-    std::string name = get_temp_image_name();
-    uint64_t size = 2 << 20;
-    uint64_t size2 = 4 << 20;
-    
-    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
-    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
-   
-    bool exists;
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_create("snap1"));
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
-    ASSERT_EQ(0, image.resize(size2));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_create("snap2"));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
-    ASSERT_EQ(0, image.snap_remove("snap1"));
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
-    ASSERT_EQ(0, image.snap_remove("snap2"));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, test_ls_snaps(image, 0));
+    value_len = sizeof(value);
   }
-
-  ioctx.close();
 }
 
-TEST_F(TestLibRBD, TestGetNameIdSnapPP)
+TEST_F(TestLibRBD, TestDeepCopyPP)
 {
+  REQUIRE_FORMAT_V2();
+
   librados::IoCtx ioctx;
-  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
+  ASSERT_EQ(0, _rados.ioctx_create(create_pool(true).c_str(), ioctx));
 
   {
     librbd::RBD rbd;
     librbd::Image image;
+    librbd::Image image2;
+    librbd::Image image3;
     int order = 0;
     std::string name = get_temp_image_name();
+    std::string name2 = get_temp_image_name();
+    std::string name3 = get_temp_image_name();
     uint64_t size = 2 << 20;
+    librbd::ImageOptions opts;
+    PrintProgress pp;
 
     ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
     ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
 
-    ASSERT_EQ(0, image.snap_create("snap1"));
-    ASSERT_EQ(0, image.snap_create("snap2"));
-    ASSERT_EQ(0, image.snap_create("snap3"));
-    vector<librbd::snap_info_t> snaps;
-    int r = image.snap_list(snaps);
-    EXPECT_TRUE(r >= 0);
-
-    for (size_t i = 0; i < snaps.size(); ++i) {
-      std::string expected_snap_name;
-      image.snap_get_name(snaps[i].id, &expected_snap_name);
-      ASSERT_EQ(expected_snap_name, snaps[i].name);
+    std::string key;
+    std::string val;
+    for (int i = 1; i <= 70; i++) {
+      key = "key" + stringify(i);
+      val = "value" + stringify(i);
+      ASSERT_EQ(0, image.metadata_set(key, val));
     }
 
-    for (size_t i = 0; i < snaps.size(); ++i) {
-      uint64_t expected_snap_id;
-      image.snap_get_id(snaps[i].name, &expected_snap_id);
-      ASSERT_EQ(expected_snap_id, snaps[i].id);
-    }
+    ASSERT_EQ(1, test_ls_pp(rbd, ioctx, 1, name.c_str()));
+    ASSERT_EQ(0, image.deep_copy(ioctx, name2.c_str(), opts));
+    ASSERT_EQ(2, test_ls_pp(rbd, ioctx, 2, name.c_str(), name2.c_str()));
+    ASSERT_EQ(0, rbd.open(ioctx, image2, name2.c_str(), NULL));
 
-    ASSERT_EQ(0, image.snap_remove("snap1"));
-    ASSERT_EQ(0, image.snap_remove("snap2"));
-    ASSERT_EQ(0, image.snap_remove("snap3"));
-    ASSERT_EQ(0, test_ls_snaps(image, 0));
-  }
+    map<string, bufferlist> pairs;
+    std::string value;
+    ASSERT_EQ(0, image2.metadata_list("", 70, &pairs));
+    ASSERT_EQ(70U, pairs.size());
 
-  ioctx.close();
-}
+    for (int i = 1; i <= 70; i++) {
+      key = "key" + stringify(i);
+      val = "value" + stringify(i);
+      ASSERT_EQ(0, image2.metadata_get(key.c_str(), &value));
+      ASSERT_STREQ(val.c_str(), value.c_str());
+    }
 
-TEST_F(TestLibRBD, TestCreateLsRenameSnapPP)
-{
-  librados::IoCtx ioctx;
-  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
+    ASSERT_EQ(0, image.deep_copy_with_progress(ioctx, name3.c_str(), opts, pp));
+    ASSERT_EQ(3, test_ls_pp(rbd, ioctx, 3, name.c_str(), name2.c_str(),
+                            name3.c_str()));
+    ASSERT_EQ(0, rbd.open(ioctx, image3, name3.c_str(), NULL));
 
-  {
-    librbd::RBD rbd;
-    librbd::Image image;
-    int order = 0;
-    std::string name = get_temp_image_name();
-    uint64_t size = 2 << 20;
-    uint64_t size2 = 4 << 20;
-    
-    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
-    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
-    
-    bool exists;
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_create("snap1"));
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
-    ASSERT_EQ(0, image.resize(size2));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_create("snap2"));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
-    ASSERT_EQ(0, image.snap_rename("snap1","snap1-rename"));
-    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1-rename", size, "snap2", size2));
-    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_exists2("snap1-rename", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(0, image.snap_remove("snap1-rename"));
-    ASSERT_EQ(0, image.snap_rename("snap2","snap2-rename"));
-    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2-rename", size2));
-    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
-    ASSERT_FALSE(exists);
-    ASSERT_EQ(0, image.snap_exists2("snap2-rename", &exists));
-    ASSERT_TRUE(exists);
-    ASSERT_EQ(0, image.snap_remove("snap2-rename"));
-    ASSERT_EQ(0, test_ls_snaps(image, 0));
+    pairs.clear();
+    ASSERT_EQ(0, image3.metadata_list("", 70, &pairs));
+    ASSERT_EQ(70U, pairs.size());
+
+    for (int i = 1; i <= 70; i++) {
+      key = "key" + stringify(i);
+      val = "value" + stringify(i);
+      ASSERT_EQ(0, image3.metadata_get(key.c_str(), &value));
+      ASSERT_STREQ(val.c_str(), value.c_str());
+    }
   }
 
   ioctx.close();
 }
 
-void simple_write_cb(rbd_completion_t cb, void *arg)
-{
-  printf("write completion cb called!\n");
-}
-
-void simple_read_cb(rbd_completion_t cb, void *arg)
-{
-  printf("read completion cb called!\n");
-}
-
-void aio_write_test_data_and_poll(rbd_image_t image, int fd, const char *test_data,
-                                  uint64_t off, size_t len, uint32_t iohint, bool *passed)
+int test_ls_snaps(rbd_image_t image, int num_expected, ...)
 {
-  rbd_completion_t comp;
-  uint64_t data = 0x123;
-  rbd_aio_create_completion((void*)&data, (rbd_callback_t) simple_write_cb, &comp);
-  printf("created completion\n");
-  printf("started write\n");
-  if (iohint)
-    rbd_aio_write2(image, off, len, test_data, comp, iohint);
-  else
-    rbd_aio_write(image, off, len, test_data, comp);
+  int num_snaps, i, j, max_size = 10;
+  va_list ap;
+  rbd_snap_info_t snaps[max_size];
+  num_snaps = rbd_snap_list(image, snaps, &max_size);
+  printf("num snaps is: %d\nexpected: %d\n", num_snaps, num_expected);
 
-  struct pollfd pfd;
-  pfd.fd = fd;
-  pfd.events = POLLIN;
+  for (i = 0; i < num_snaps; i++) {
+    printf("snap: %s\n", snaps[i].name);
+  }
 
-  ASSERT_EQ(1, poll(&pfd, 1, -1));
-  ASSERT_TRUE(pfd.revents & POLLIN);
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    uint64_t expected_size = va_arg(ap, uint64_t);
+    bool found = false;
+    for (j = 0; j < num_snaps; j++) {
+      if (snaps[j].name == NULL)
+       continue;
+      if (strcmp(snaps[j].name, expected) == 0) {
+       printf("found %s with size %llu\n", snaps[j].name, (unsigned long long) snaps[j].size);
+       EXPECT_EQ(expected_size, snaps[j].size);
+       free((void *) snaps[j].name);
+       snaps[j].name = NULL;
+       found = true;
+       break;
+      }
+    }
+    EXPECT_TRUE(found);
+  }
+  va_end(ap);
 
-  rbd_completion_t comps[1];
-  ASSERT_EQ(1, rbd_poll_io_events(image, comps, 1));
-  uint64_t count;
-  ASSERT_EQ(static_cast<ssize_t>(sizeof(count)),
-            read(fd, &count, sizeof(count)));
-  int r = rbd_aio_get_return_value(comps[0]);
-  ASSERT_TRUE(rbd_aio_is_complete(comps[0]));
-  ASSERT_TRUE(*(uint64_t*)rbd_aio_get_arg(comps[0]) == data);
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(0, r);
-  printf("finished write\n");
-  rbd_aio_release(comps[0]);
-  *passed = true;
-}
+  for (i = 0; i < num_snaps; i++) {
+    EXPECT_EQ((const char *)0, snaps[i].name);
+  }
 
-void aio_write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len, uint32_t iohint, bool *passed)
-{
-  rbd_completion_t comp;
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
-  printf("created completion\n");
-  if (iohint)
-    rbd_aio_write2(image, off, len, test_data, comp, iohint);
-  else
-    rbd_aio_write(image, off, len, test_data, comp);
-  printf("started write\n");
-  rbd_aio_wait_for_complete(comp);
-  int r = rbd_aio_get_return_value(comp);
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(0, r);
-  printf("finished write\n");
-  rbd_aio_release(comp);
-  *passed = true;
+  return num_snaps;
 }
 
-void write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+TEST_F(TestLibRBD, TestCreateLsDeleteSnap)
 {
-  ssize_t written;
-  if (iohint)
-    written = rbd_write2(image, off, len, test_data, iohint);
-  else
-    written = rbd_write(image, off, len, test_data);
-  printf("wrote: %d\n", (int) written);
-  ASSERT_EQ(len, static_cast<size_t>(written));
-  *passed = true;
-}
+  rados_ioctx_t ioctx;
+  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
 
-void aio_discard_test_data(rbd_image_t image, uint64_t off, uint64_t len, bool *passed)
-{
-  rbd_completion_t comp;
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
-  rbd_aio_discard(image, off, len, comp);
-  rbd_aio_wait_for_complete(comp);
-  int r = rbd_aio_get_return_value(comp);
-  ASSERT_EQ(0, r);
-  printf("aio discard: %d~%d = %d\n", (int)off, (int)len, (int)r);
-  rbd_aio_release(comp);
-  *passed = true;
+  rbd_image_t image;
+  int order = 0;
+  std::string name = get_temp_image_name();
+  uint64_t size = 2 << 20;
+  uint64_t size2 = 4 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
+
+  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
+  ASSERT_EQ(0, rbd_resize(image, size2));
+  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
+  ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
+  ASSERT_EQ(0, rbd_snap_remove(image, "snap1"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
+  ASSERT_EQ(0, rbd_snap_remove(image, "snap2"));
+  ASSERT_EQ(0, test_ls_snaps(image, 0));
+  
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
 }
 
-void discard_test_data(rbd_image_t image, uint64_t off, size_t len, bool *passed)
+int test_get_snapshot_timestamp(rbd_image_t image, uint64_t snap_id)
 {
-  ssize_t written;
-  written = rbd_discard(image, off, len);
-  printf("discard: %d~%d = %d\n", (int)off, (int)len, (int)written);
-  ASSERT_EQ(len, static_cast<size_t>(written));
-  *passed = true;
+  struct timespec timestamp;
+  EXPECT_EQ(0, rbd_snap_get_timestamp(image, snap_id, &timestamp));
+  EXPECT_LT(0, timestamp.tv_sec);
+  return 0;
 }
 
-void aio_read_test_data_and_poll(rbd_image_t image, int fd, const char *expected,
-                                 uint64_t off, size_t len, uint32_t iohint, bool *passed)
+TEST_F(TestLibRBD, TestGetSnapShotTimeStamp)
 {
-  rbd_completion_t comp;
-  char *result = (char *)malloc(len + 1);
+  REQUIRE_FORMAT_V2();
 
-  ASSERT_NE(static_cast<char *>(NULL), result);
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
-  printf("created completion\n");
-  printf("started read\n");
-  if (iohint)
-    rbd_aio_read2(image, off, len, result, comp, iohint);
-  else
-    rbd_aio_read(image, off, len, result, comp);
+  rados_ioctx_t ioctx;
+  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
 
-  struct pollfd pfd;
-  pfd.fd = fd;
-  pfd.events = POLLIN;
+  rbd_image_t image;
+  int order = 0;
+  std::string name = get_temp_image_name();
+  uint64_t size = 2 << 20;
+  int num_snaps, max_size = 10;
+  rbd_snap_info_t snaps[max_size];
 
-  ASSERT_EQ(1, poll(&pfd, 1, -1));
-  ASSERT_TRUE(pfd.revents & POLLIN);
+  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
 
-  rbd_completion_t comps[1];
-  ASSERT_EQ(1, rbd_poll_io_events(image, comps, 1));
-  uint64_t count;
-  ASSERT_EQ(static_cast<ssize_t>(sizeof(count)),
-            read(fd, &count, sizeof(count)));
+  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
+  num_snaps = rbd_snap_list(image, snaps, &max_size);
+  ASSERT_EQ(1, num_snaps);
+  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[0].id));
+  free((void *)snaps[0].name);
 
-  int r = rbd_aio_get_return_value(comps[0]);
-  ASSERT_TRUE(rbd_aio_is_complete(comps[0]));
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(len, static_cast<size_t>(r));
-  rbd_aio_release(comps[0]);
-  if (memcmp(result, expected, len)) {
-    printf("read: %s\nexpected: %s\n", result, expected);
-    ASSERT_EQ(0, memcmp(result, expected, len));
-  }
-  free(result);
-  *passed = true;
+  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
+  num_snaps = rbd_snap_list(image, snaps, &max_size);
+  ASSERT_EQ(2, num_snaps);
+  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[0].id));
+  ASSERT_EQ(0, test_get_snapshot_timestamp(image, snaps[1].id));
+  free((void *)snaps[0].name);
+  free((void *)snaps[1].name);
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
 }
 
-void aio_read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len, uint32_t iohint, bool *passed)
+
+int test_ls_snaps(librbd::Image& image, size_t num_expected, ...)
 {
-  rbd_completion_t comp;
-  char *result = (char *)malloc(len + 1);
+  int r;
+  size_t i, j;
+  va_list ap;
+  vector<librbd::snap_info_t> snaps;
+  r = image.snap_list(snaps);
+  EXPECT_TRUE(r >= 0);
+  cout << "num snaps is: " << snaps.size() << std::endl
+           << "expected: " << num_expected << std::endl;
 
-  ASSERT_NE(static_cast<char *>(NULL), result);
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
-  printf("created completion\n");
-  if (iohint)
-    rbd_aio_read2(image, off, len, result, comp, iohint);
-  else
-    rbd_aio_read(image, off, len, result, comp);
-  printf("started read\n");
-  rbd_aio_wait_for_complete(comp);
-  int r = rbd_aio_get_return_value(comp);
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(len, static_cast<size_t>(r));
-  rbd_aio_release(comp);
-  if (memcmp(result, expected, len)) {
-    printf("read: %s\nexpected: %s\n", result, expected);
-    ASSERT_EQ(0, memcmp(result, expected, len));
+  for (i = 0; i < snaps.size(); i++) {
+    cout << "snap: " << snaps[i].name << std::endl;
   }
-  free(result);
-  *passed = true;
-}
 
-void read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len, uint32_t iohint, bool *passed)
-{
-  ssize_t read;
-  char *result = (char *)malloc(len + 1);
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    uint64_t expected_size = va_arg(ap, uint64_t);
+    int found = 0;
+    for (j = 0; j < snaps.size(); j++) {
+      if (snaps[j].name == "")
+       continue;
+      if (strcmp(snaps[j].name.c_str(), expected) == 0) {
+       cout << "found " << snaps[j].name << " with size " << snaps[j].size
+            << std::endl;
+       EXPECT_EQ(expected_size, snaps[j].size);
+       snaps[j].name = "";
+       found = 1;
+       break;
+      }
+    }
+    EXPECT_TRUE(found);
+  }
+  va_end(ap);
 
-  ASSERT_NE(static_cast<char *>(NULL), result);
-  if (iohint)
-    read = rbd_read2(image, off, len, result, iohint);
-  else
-    read = rbd_read(image, off, len, result);
-  printf("read: %d\n", (int) read);
-  ASSERT_EQ(len, static_cast<size_t>(read));
-  result[len] = '\0';
-  if (memcmp(result, expected, len)) {
-    printf("read: %s\nexpected: %s\n", result, expected);
-    ASSERT_EQ(0, memcmp(result, expected, len));
+  for (i = 0; i < snaps.size(); i++) {
+    EXPECT_EQ("", snaps[i].name);
   }
-  free(result);
-  *passed = true;
+
+  return snaps.size();
 }
 
-void aio_writesame_test_data(rbd_image_t image, const char *test_data, uint64_t off, uint64_t len,
-                             uint64_t data_len, uint32_t iohint, bool *passed)
+TEST_F(TestLibRBD, TestCreateLsDeleteSnapPP)
 {
-  rbd_completion_t comp;
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
-  printf("created completion\n");
-  int r;
-  r = rbd_aio_writesame(image, off, len, test_data, data_len, comp, iohint);
-  printf("started writesame\n");
-  if (len % data_len) {
-    ASSERT_EQ(-EINVAL, r);
-    printf("expected fail, finished writesame\n");
-    rbd_aio_release(comp);
-    *passed = true;
-    return;
-  }
-
-  rbd_aio_wait_for_complete(comp);
-  r = rbd_aio_get_return_value(comp);
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(0, r);
-  printf("finished writesame\n");
-  rbd_aio_release(comp);
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
 
-  //verify data
-  printf("to verify the data\n");
-  ssize_t read;
-  char *result = (char *)malloc(data_len+ 1);
-  ASSERT_NE(static_cast<char *>(NULL), result);
-  uint64_t left = len;
-  while (left > 0) {
-    read = rbd_read(image, off, data_len, result);
-    ASSERT_EQ(data_len, static_cast<size_t>(read));
-    result[data_len] = '\0';
-    if (memcmp(result, test_data, data_len)) {
-      printf("read: %d ~ %d\n", (int) off, (int) read);
-      printf("read: %s\nexpected: %s\n", result, test_data);
-      ASSERT_EQ(0, memcmp(result, test_data, data_len));
-    }
-    off += data_len;
-    left -= data_len;
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    std::string name = get_temp_image_name();
+    uint64_t size = 2 << 20;
+    uint64_t size2 = 4 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+   
+    bool exists;
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_create("snap1"));
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
+    ASSERT_EQ(0, image.resize(size2));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_create("snap2"));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
+    ASSERT_EQ(0, image.snap_remove("snap1"));
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
+    ASSERT_EQ(0, image.snap_remove("snap2"));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, test_ls_snaps(image, 0));
   }
-  ASSERT_EQ(0U, left);
-  free(result);
-  printf("verified\n");
 
-  *passed = true;
+  ioctx.close();
 }
 
-void writesame_test_data(rbd_image_t image, const char *test_data, uint64_t off, uint64_t len,
-                         uint64_t data_len, uint32_t iohint, bool *passed)
+TEST_F(TestLibRBD, TestGetNameIdSnapPP)
 {
-  ssize_t written;
-  written = rbd_writesame(image, off, len, test_data, data_len, iohint);
-  if (len % data_len) {
-    ASSERT_EQ(-EINVAL, written);
-    printf("expected fail, finished writesame\n");
-    *passed = true;
-    return;
-  }
-  ASSERT_EQ(len, static_cast<size_t>(written));
-  printf("wrote: %d\n", (int) written);
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
 
-  //verify data
-  printf("to verify the data\n");
-  ssize_t read;
-  char *result = (char *)malloc(data_len+ 1);
-  ASSERT_NE(static_cast<char *>(NULL), result);
-  uint64_t left = len;
-  while (left > 0) {
-    read = rbd_read(image, off, data_len, result);
-    ASSERT_EQ(data_len, static_cast<size_t>(read));
-    result[data_len] = '\0';
-    if (memcmp(result, test_data, data_len)) {
-      printf("read: %d ~ %d\n", (int) off, (int) read);
-      printf("read: %s\nexpected: %s\n", result, test_data);
-      ASSERT_EQ(0, memcmp(result, test_data, data_len));
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    std::string name = get_temp_image_name();
+    uint64_t size = 2 << 20;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+
+    ASSERT_EQ(0, image.snap_create("snap1"));
+    ASSERT_EQ(0, image.snap_create("snap2"));
+    ASSERT_EQ(0, image.snap_create("snap3"));
+    vector<librbd::snap_info_t> snaps;
+    int r = image.snap_list(snaps);
+    EXPECT_TRUE(r >= 0);
+
+    for (size_t i = 0; i < snaps.size(); ++i) {
+      std::string expected_snap_name;
+      image.snap_get_name(snaps[i].id, &expected_snap_name);
+      ASSERT_EQ(expected_snap_name, snaps[i].name);
     }
-    off += data_len;
-    left -= data_len;
+
+    for (size_t i = 0; i < snaps.size(); ++i) {
+      uint64_t expected_snap_id;
+      image.snap_get_id(snaps[i].name, &expected_snap_id);
+      ASSERT_EQ(expected_snap_id, snaps[i].id);
+    }
+
+    ASSERT_EQ(0, image.snap_remove("snap1"));
+    ASSERT_EQ(0, image.snap_remove("snap2"));
+    ASSERT_EQ(0, image.snap_remove("snap3"));
+    ASSERT_EQ(0, test_ls_snaps(image, 0));
   }
-  ASSERT_EQ(0U, left);
-  free(result);
-  printf("verified\n");
 
-  *passed = true;
+  ioctx.close();
 }
 
-void aio_compare_and_write_test_data(rbd_image_t image, const char *cmp_data,
-                                     const char *test_data, uint64_t off,
-                                     size_t len, uint32_t iohint, bool *passed)
+TEST_F(TestLibRBD, TestCreateLsRenameSnapPP)
 {
-  rbd_completion_t comp;
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
-  printf("created completion\n");
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx));
 
-  uint64_t mismatch_offset;
-  rbd_aio_compare_and_write(image, off, len, cmp_data, test_data, comp, &mismatch_offset, iohint);
-  printf("started aio compare and write\n");
-  rbd_aio_wait_for_complete(comp);
-  int r = rbd_aio_get_return_value(comp);
-  printf("return value is: %d\n", r);
-  ASSERT_EQ(0, r);
-  printf("finished aio compare and write\n");
-  rbd_aio_release(comp);
-  *passed = true;
-}
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    std::string name = get_temp_image_name();
+    uint64_t size = 2 << 20;
+    uint64_t size2 = 4 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+    
+    bool exists;
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_create("snap1"));
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
+    ASSERT_EQ(0, image.resize(size2));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_create("snap2"));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
+    ASSERT_EQ(0, image.snap_rename("snap1","snap1-rename"));
+    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1-rename", size, "snap2", size2));
+    ASSERT_EQ(0, image.snap_exists2("snap1", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_exists2("snap1-rename", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(0, image.snap_remove("snap1-rename"));
+    ASSERT_EQ(0, image.snap_rename("snap2","snap2-rename"));
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2-rename", size2));
+    ASSERT_EQ(0, image.snap_exists2("snap2", &exists));
+    ASSERT_FALSE(exists);
+    ASSERT_EQ(0, image.snap_exists2("snap2-rename", &exists));
+    ASSERT_TRUE(exists);
+    ASSERT_EQ(0, image.snap_remove("snap2-rename"));
+    ASSERT_EQ(0, test_ls_snaps(image, 0));
+  }
 
-void compare_and_write_test_data(rbd_image_t image, const char *cmp_data,
-                                 const char *test_data, uint64_t off, size_t len,
-                                 uint64_t *mismatch_off, uint32_t iohint, bool *passed)
-{
-  printf("start compare and write\n");
-  ssize_t written;
-  written = rbd_compare_and_write(image, off, len, cmp_data, test_data, mismatch_off, iohint);
-  printf("compare and  wrote: %d\n", (int) written);
-  ASSERT_EQ(len, static_cast<size_t>(written));
-  *passed = true;
+  ioctx.close();
 }
 
-
 TEST_F(TestLibRBD, TestIO)
 {
   rados_ioctx_t ioctx;
@@ -1971,111 +2103,92 @@ TEST_F(TestLibRBD, TestIO)
   ASSERT_EQ(0, rados_conf_set(_cluster, "rbd_read_from_replica_policy", "balance"));
   ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
 
-  bool skip_discard = is_skip_partial_discard_enabled(image);
+  test_io(image);
 
-  char test_data[TEST_IO_SIZE + 1];
-  char zero_data[TEST_IO_SIZE + 1];
-  char mismatch_data[TEST_IO_SIZE + 1];
-  int i;
-  uint64_t mismatch_offset;
+  ASSERT_EQ(0, rbd_close(image));
 
-  for (i = 0; i < TEST_IO_SIZE; ++i) {
-    test_data[i] = (char) (rand() % (126 - 33) + 33);
-  }
-  test_data[TEST_IO_SIZE] = '\0';
-  memset(zero_data, 0, sizeof(zero_data));
-  memset(mismatch_data, 9, sizeof(mismatch_data));
+  rados_ioctx_destroy(ioctx);
+}
 
-  for (i = 0; i < 5; ++i)
-    ASSERT_PASSED(write_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+TEST_F(TestLibRBD, TestEncryptionLUKS1)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_JOURNALING));
 
-  for (i = 5; i < 10; ++i)
-    ASSERT_PASSED(aio_write_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+  rados_ioctx_t ioctx;
+  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
 
-  for (i = 0; i < 5; ++i)
-    ASSERT_PASSED(compare_and_write_test_data, image, test_data, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, &mismatch_offset, 0);
+  int order = 0;
+  std::string name = get_temp_image_name();
+  uint64_t size = 32 << 20;
 
-  for (i = 5; i < 10; ++i)
-    ASSERT_PASSED(aio_compare_and_write_test_data, image, test_data, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rados_conf_set(
+          _cluster, "rbd_read_from_replica_policy", "balance"));
 
-  for (i = 0; i < 5; ++i)
-    ASSERT_PASSED(read_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+  rbd_image_t image;
+  rbd_encryption_luks1_format_options_t opts = {
+          .alg = RBD_ENCRYPTION_ALGORITHM_AES256,
+          .passphrase = "password",
+          .passphrase_size = 8,
+  };
+  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
 
-  for (i = 5; i < 10; ++i)
-    ASSERT_PASSED(aio_read_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE, 0);
+#ifndef HAVE_LIBCRYPTSETUP
+  ASSERT_EQ(-ENOTSUP, rbd_encryption_format(
+          image, RBD_ENCRYPTION_FORMAT_LUKS1, &opts, sizeof(opts)));
+  ASSERT_EQ(-ENOTSUP, rbd_encryption_load(
+          image, RBD_ENCRYPTION_FORMAT_LUKS1, &opts, sizeof(opts)));
+#else
+  ASSERT_EQ(0, rbd_encryption_format(
+          image, RBD_ENCRYPTION_FORMAT_LUKS1, &opts, sizeof(opts)));
+  ASSERT_EQ(0, rbd_encryption_load(
+          image, RBD_ENCRYPTION_FORMAT_LUKS1, &opts, sizeof(opts)));
+
+  test_io(image);
+#endif
 
-  // discard 2nd, 4th sections.
-  ASSERT_PASSED(discard_test_data, image, TEST_IO_SIZE, TEST_IO_SIZE);
-  ASSERT_PASSED(aio_discard_test_data, image, TEST_IO_SIZE*3, TEST_IO_SIZE);
+  ASSERT_EQ(0, rbd_close(image));
+  rados_ioctx_destroy(ioctx);
+}
 
-  ASSERT_PASSED(read_test_data, image, test_data,  0, TEST_IO_SIZE, 0);
-  ASSERT_PASSED(read_test_data, image, skip_discard ? test_data : zero_data,
-               TEST_IO_SIZE, TEST_IO_SIZE, 0);
-  ASSERT_PASSED(read_test_data, image, test_data,  TEST_IO_SIZE*2, TEST_IO_SIZE, 0);
-  ASSERT_PASSED(read_test_data, image, skip_discard ? test_data : zero_data,
-               TEST_IO_SIZE*3, TEST_IO_SIZE, 0);
-  ASSERT_PASSED(read_test_data, image, test_data,  TEST_IO_SIZE*4, TEST_IO_SIZE, 0);
+TEST_F(TestLibRBD, TestEncryptionLUKS2)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_JOURNALING));
 
-  for (i = 0; i < 15; ++i) {
-    if (i % 3 == 2) {
-      ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
-    } else if (i % 3 == 1) {
-      ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-    } else {
-      ASSERT_PASSED(writesame_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(writesame_test_data, image, zero_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-    }
-  }
-  for (i = 0; i < 15; ++i) {
-    if (i % 3 == 2) {
-      ASSERT_PASSED(aio_writesame_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(aio_writesame_test_data, image, zero_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32 + i, TEST_IO_SIZE, 0);
-    } else if (i % 3 == 1) {
-      ASSERT_PASSED(aio_writesame_test_data, image, test_data, TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(aio_writesame_test_data, image, zero_data, TEST_IO_SIZE + i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-    } else {
-      ASSERT_PASSED(aio_writesame_test_data, image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-      ASSERT_PASSED(aio_writesame_test_data, image, zero_data, TEST_IO_SIZE * i, TEST_IO_SIZE * i * 32, TEST_IO_SIZE, 0);
-    }
-  }
+  rados_ioctx_t ioctx;
+  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
 
-  rbd_image_info_t info;
-  rbd_completion_t comp;
-  ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
-  // can't read or write starting past end
-  ASSERT_EQ(-EINVAL, rbd_write(image, info.size, 1, test_data));
-  ASSERT_EQ(-EINVAL, rbd_read(image, info.size, 1, test_data));
-  // reading through end returns amount up to end
-  ASSERT_EQ(10, rbd_read(image, info.size - 10, 100, test_data));
-  // writing through end returns amount up to end
-  ASSERT_EQ(10, rbd_write(image, info.size - 10, 100, test_data));
+  int order = 0;
+  std::string name = get_temp_image_name();
+  uint64_t size = 32 << 20;
 
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
-  ASSERT_EQ(0, rbd_aio_write(image, info.size, 1, test_data, comp));
-  ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
-  ASSERT_EQ(-EINVAL, rbd_aio_get_return_value(comp));
-  rbd_aio_release(comp);
+  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rados_conf_set(
+          _cluster, "rbd_read_from_replica_policy", "balance"));
 
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
-  ASSERT_EQ(0, rbd_aio_read(image, info.size, 1, test_data, comp));
-  ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
-  ASSERT_EQ(-EINVAL, rbd_aio_get_return_value(comp));
-  rbd_aio_release(comp);
+  rbd_image_t image;
+  rbd_encryption_luks2_format_options_t opts = {
+          .alg = RBD_ENCRYPTION_ALGORITHM_AES256,
+          .passphrase = "password",
+          .passphrase_size = 8,
+  };
+  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
 
-  ASSERT_PASSED(write_test_data, image, zero_data, 0, TEST_IO_SIZE, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
-  ASSERT_EQ(-EILSEQ, rbd_compare_and_write(image, 0, TEST_IO_SIZE, mismatch_data, mismatch_data, &mismatch_offset, 0));
-  ASSERT_EQ(0U, mismatch_offset);
-  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
-  ASSERT_EQ(0, rbd_aio_compare_and_write(image, 0, TEST_IO_SIZE, mismatch_data, mismatch_data, comp, &mismatch_offset, 0));
-  ASSERT_EQ(0, rbd_aio_wait_for_complete(comp));
-  ASSERT_EQ(0U, mismatch_offset);
-  rbd_aio_release(comp);
+#ifndef HAVE_LIBCRYPTSETUP
+  ASSERT_EQ(-ENOTSUP, rbd_encryption_format(
+          image, RBD_ENCRYPTION_FORMAT_LUKS2, &opts, sizeof(opts)));
+  ASSERT_EQ(-ENOTSUP, rbd_encryption_load(
+          image, RBD_ENCRYPTION_FORMAT_LUKS2, &opts, sizeof(opts)));
+#else
+  ASSERT_EQ(0, rbd_encryption_format(
+          image, RBD_ENCRYPTION_FORMAT_LUKS2, &opts, sizeof(opts)));
+  ASSERT_EQ(0, rbd_encryption_load(
+          image, RBD_ENCRYPTION_FORMAT_LUKS2, &opts, sizeof(opts)));
+
+  test_io(image);
+#endif
 
-  ASSERT_PASSED(validate_object_map, image);
   ASSERT_EQ(0, rbd_close(image));
-
   rados_ioctx_destroy(ioctx);
 }