librbd/migration: make ImageDispatch handle encryption for non-native formats

With NativeFormat now being handled via dispatch, handling encryption
for non-native formats (i.e. mapping to raw image extents and performing
decryption/mapping back on completion) in the migration layer is really
straightforward.

Note that alignment doesn't need to be performed in the migration layer
because it happens on the destination image -- the "align and resubmit"
logic in C_UnalignedObjectReadRequest should kick in before the call to
read_parent().

Fixes: https://tracker.ceph.com/issues/53674
Co-authored-by: Or Ozeri <oro@il.ibm.com>
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/qa/workunits/rbd/luks-encryption.sh b/qa/workunits/rbd/luks-encryption.sh
index c716a3de3a618807e888f93ef342f2e37bdd5ab6..97cb5a0fe87ef4b15f71ae9f9eec3a267ba0991f 100755 (executable)
--- a/qa/workunits/rbd/luks-encryption.sh
+++ b/qa/workunits/rbd/luks-encryption.sh
@@ -2,7 +2,7 @@
 set -ex
 
 CEPH_ID=${CEPH_ID:-admin}
-TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata"
+TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata /tmp/rawexport /tmp/export.qcow2"
 
 _sudo()
 {
@@ -154,6 +154,172 @@ function test_plaintext_detection {
   test_clone_and_load_with_a_single_passphrase false
 }
 
+function test_migration_read_and_copyup() {
+  cp /tmp/testdata2 /tmp/cmpdata
+
+  # test reading
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+
+  # trigger copyup at the beginning and at the end
+  xfs_io -c 'pwrite -S 0xab -W 0 4k' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xba -W 4095k 4k' $LIBRBD_DEV /tmp/cmpdata
+
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping after migration is executed
+  rbd migration execute testimg1
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping after migration is committed
+  rbd migration commit testimg1
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+}
+
+function test_migration_native_with_snaps() {
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/testdata1
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/testdata2
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  test_migration_read_and_copyup
+
+  # check that snapshots aren't affected by copyups
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/testdata1
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/testdata2
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  rbd snap rm testimg1@snap2
+  rbd snap rm testimg1@snap1
+  rbd rm testimg1
+}
+
+function test_migration() {
+  local format=$1
+
+  rbd encryption format testimg $format /tmp/passphrase
+
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  dd if=/tmp/testdata1 of=$LIBRBD_DEV conv=fsync bs=1M
+  rbd snap create testimg@snap1
+  dd if=/tmp/testdata2 of=$LIBRBD_DEV conv=fsync bs=1M
+  rbd snap create testimg@snap2
+  # FIXME: https://tracker.ceph.com/issues/67401
+  # leave HEAD with the same data as snap2 as a workaround
+  # dd if=/tmp/testdata3 of=$LIBRBD_DEV conv=fsync bs=1M
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # live import a raw image
+  rbd export testimg /tmp/rawexport
+  rbd migration prepare --import-only --source-spec '{"type": "raw", "stream": {"type": "file", "file_path": "/tmp/rawexport"}}' testimg1
+  test_migration_read_and_copyup
+  rbd rm testimg1
+
+  # live import a qcow image
+  qemu-img convert -f raw -O qcow2 /tmp/rawexport /tmp/export.qcow2
+  rbd migration prepare --import-only --source-spec '{"type": "qcow", "stream": {"type": "file", "file_path": "/tmp/export.qcow2"}}' testimg1
+  test_migration_read_and_copyup
+  rbd rm testimg1
+
+  # live import a native image
+  rbd migration prepare --import-only testimg@snap2 testimg1
+  test_migration_native_with_snaps
+
+  # live migrate a native image (removes testimg)
+  rbd migration prepare testimg testimg1
+  test_migration_native_with_snaps
+
+  rm /tmp/rawexport /tmp/export.qcow2
+}
+
+function test_migration_clone() {
+  local format=$1
+
+  truncate -s 0 /tmp/cmpdata
+  truncate -s 32M /tmp/cmpdata
+
+  rbd encryption format testimg $format /tmp/passphrase
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase)
+  xfs_io -c 'pwrite -S 0xaa -W 4M 1M' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xaa -W 14M 1M' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xaa -W 25M 1M' $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  rbd snap create testimg@snap
+  rbd snap protect testimg@snap
+  rbd clone testimg@snap testimg1
+
+  rbd encryption format testimg1 $format /tmp/passphrase2
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
+  xfs_io -c 'pwrite -S 0xbb -W 2M 1M' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xbb -W 19M 1M' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xbb -W 28M 1M' $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # FIXME: https://tracker.ceph.com/issues/67402
+  rbd config image set testimg1 rbd_sparse_read_threshold_bytes 1
+
+  # live migrate a native clone image (removes testimg1)
+  rbd migration prepare testimg1 testimg2
+
+  # test reading
+  # FIXME: https://tracker.ceph.com/issues/63184
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+
+  # trigger copyup for an unwritten area
+  xfs_io -c 'pwrite -S 0xcc -W 24167k 4k' $LIBRBD_DEV /tmp/cmpdata
+
+  # trigger copyup for areas written in testimg (parent)
+  xfs_io -c 'pwrite -S 0xcc -W 4245k 4k' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xcc -W 13320k 4k' $LIBRBD_DEV /tmp/cmpdata
+
+  # trigger copyup for areas written in testimg1 (clone)
+  xfs_io -c 'pwrite -S 0xcc -W 2084k 4k' $LIBRBD_DEV /tmp/cmpdata
+  xfs_io -c 'pwrite -S 0xcc -W 32612k 4k' $LIBRBD_DEV /tmp/cmpdata
+
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping
+  # FIXME: https://tracker.ceph.com/issues/63184
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping after migration is executed
+  rbd migration execute testimg2
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  # test reading on a fresh mapping after migration is committed
+  rbd migration commit testimg2
+  LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase)
+  cmp $LIBRBD_DEV /tmp/cmpdata
+  _sudo rbd device unmap -t nbd $LIBRBD_DEV
+
+  rbd rm testimg2
+  rbd snap unprotect testimg@snap
+  rbd snap rm testimg@snap
+  rbd rm testimg
+}
+
 function get_nbd_device_paths {
   rbd device list -t nbd | tail -n +2 | egrep "\s+rbd\s+testimg" | awk '{print $5;}'
 }
@@ -169,10 +335,16 @@ function clean_up {
     _sudo rbd device unmap -t nbd $device
   done
 
+  rbd migration abort testimg2 || true
   rbd remove testimg2 || true
+  rbd migration abort testimg1 || true
+  rbd snap remove testimg1@snap2 || true
+  rbd snap remove testimg1@snap1 || true
   rbd snap unprotect testimg1@snap || true
   rbd snap remove testimg1@snap || true
   rbd remove testimg1 || true
+  rbd snap remove testimg@snap2 || true
+  rbd snap remove testimg@snap1 || true
   rbd snap unprotect testimg@snap || true
   rbd snap remove testimg@snap || true
   rbd remove testimg || true
@@ -217,4 +389,16 @@ test_clone_encryption
 _sudo rbd device unmap -t nbd $RAW_DEV
 rbd rm testimg
 
+rbd create --size 20M testimg
+test_migration luks1
+
+rbd create --size 32M testimg
+test_migration luks2
+
+rbd create --size 36M testimg
+test_migration_clone luks1
+
+rbd create --size 48M testimg
+test_migration_clone luks2
+
 echo OK

diff --git a/src/librbd/migration/ImageDispatch.cc b/src/librbd/migration/ImageDispatch.cc
index 20cc3cb07e7eb79a74505a2514fc53feba625d2f..8420d06d1960bd496b335d5ee0e8c692e2b87b99 100644 (file)
--- a/src/librbd/migration/ImageDispatch.cc
+++ b/src/librbd/migration/ImageDispatch.cc
@@ -5,8 +5,12 @@
 #include "include/neorados/RADOS.hpp"
 #include "common/dout.h"
 #include "librbd/ImageCtx.h"
+#include "librbd/crypto/CryptoInterface.h"
+#include "librbd/crypto/EncryptionFormat.h"
 #include "librbd/io/AioCompletion.h"
+#include "librbd/io/Utils.h"
 #include "librbd/migration/FormatInterface.h"
+#include "librbd/Utils.h"
 
 #define dout_subsys ceph_subsys_rbd
 #undef dout_prefix
@@ -16,6 +20,72 @@
 namespace librbd {
 namespace migration {
 
+namespace {
+
+struct C_DecryptData : public io::ReadResult::C_ImageReadRequest {
+  crypto::CryptoInterface* crypto;
+
+  C_DecryptData(io::AioCompletion* aio_comp, const io::Extents& image_extents,
+                crypto::CryptoInterface* crypto)
+      : C_ImageReadRequest(aio_comp, 0, image_extents), crypto(crypto) {}
+
+  void finish(int r) override {
+    if (r < 0) {
+      C_ImageReadRequest::finish(r);
+      return;
+    }
+
+    auto ciphertext_bl = std::move(bl);
+    for (const auto& extent : image_extents) {
+      ceph::bufferlist tmp;
+      ciphertext_bl.splice(0, extent.second, &tmp);
+      int r = crypto->decrypt(&tmp, extent.first);
+      if (r < 0) {
+        C_ImageReadRequest::finish(r);
+        return;
+      }
+      bl.claim_append(tmp);
+    }
+
+    C_ImageReadRequest::finish(0);
+  }
+};
+
+template <typename I>
+struct C_MapSnapshotDelta : public io::C_AioRequest {
+  io::SnapshotDelta* snapshot_delta;
+  I* image_ctx;
+
+  C_MapSnapshotDelta(io::AioCompletion* aio_comp,
+                     io::SnapshotDelta* snapshot_delta, I* image_ctx)
+      : C_AioRequest(aio_comp), snapshot_delta(snapshot_delta),
+        image_ctx(image_ctx) {}
+
+  void finish(int r) override {
+    if (r < 0) {
+      C_AioRequest::finish(r);
+      return;
+    }
+
+    auto raw_snapshot_delta = std::move(*snapshot_delta);
+    for (const auto& [key, raw_sparse_extents] : raw_snapshot_delta) {
+      auto& sparse_extents = (*snapshot_delta)[key];
+      for (const auto& raw_sparse_extent : raw_sparse_extents) {
+        auto off = io::util::raw_to_area_offset(*image_ctx,
+                                                raw_sparse_extent.get_off());
+        ceph_assert(off.second == io::ImageArea::DATA);
+        sparse_extents.insert(off.first, raw_sparse_extent.get_len(),
+                              {raw_sparse_extent.get_val().state,
+                               raw_sparse_extent.get_len()});
+      }
+    }
+
+    C_AioRequest::finish(0);
+  }
+};
+
+} // anonymous namespace
+
 template <typename I>
 ImageDispatch<I>::ImageDispatch(I* image_ctx,
                                 std::unique_ptr<FormatInterface> format)
@@ -50,6 +120,33 @@ bool ImageDispatch<I>::read(
   }
 
   *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
+
+  if (m_image_ctx->encryption_format != nullptr &&
+      (*image_dispatch_flags & io::IMAGE_DISPATCH_FLAG_CRYPTO_HEADER) == 0) {
+    auto crypto = m_image_ctx->encryption_format->get_crypto();
+
+    // alignment should be performed on the destination image (see
+    // C_UnalignedObjectReadRequest in CryptoObjectDispatch)
+    for (const auto& extent : image_extents) {
+      ceph_assert(crypto->is_aligned(extent.first, extent.second));
+    }
+
+    aio_comp->read_result = std::move(read_result);
+    aio_comp->read_result.set_image_extents(image_extents);
+
+    aio_comp->set_request_count(1);
+    auto ctx = new C_DecryptData(aio_comp, image_extents, crypto);
+    aio_comp = io::AioCompletion::create_and_start<Context>(
+        ctx, util::get_image_ctx(m_image_ctx), io::AIO_TYPE_READ);
+    read_result = io::ReadResult(&ctx->bl);
+
+    // map to raw image extents _after_ DATA area extents are captured
+    for (auto& extent : image_extents) {
+      extent.first = io::util::area_to_raw_offset(*m_image_ctx, extent.first,
+                                                  io::ImageArea::DATA);
+    }
+  }
+
   m_format->read(aio_comp, io_context->get_read_snap(),
                  std::move(image_extents), std::move(read_result),
                  op_flags, read_flags, parent_trace);
@@ -148,7 +245,20 @@ bool ImageDispatch<I>::list_snaps(
   *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
 
   aio_comp->set_request_count(1);
-  auto ctx = new io::C_AioRequest(aio_comp);
+  Context* ctx;
+
+  if (m_image_ctx->encryption_format != nullptr &&
+      (*image_dispatch_flags & io::IMAGE_DISPATCH_FLAG_CRYPTO_HEADER) == 0) {
+    // map to raw image extents
+    for (auto& extent : image_extents) {
+      extent.first = io::util::area_to_raw_offset(*m_image_ctx, extent.first,
+                                                  io::ImageArea::DATA);
+    }
+    // ... and back on completion
+    ctx = new C_MapSnapshotDelta(aio_comp, snapshot_delta, m_image_ctx);
+  } else {
+    ctx = new io::C_AioRequest(aio_comp);
+  }
 
   m_format->list_snaps(std::move(image_extents), std::move(snap_ids),
                        list_snaps_flags, snapshot_delta, parent_trace,

diff --git a/src/librbd/migration/OpenSourceImageRequest.cc b/src/librbd/migration/OpenSourceImageRequest.cc
index b22900795b9d0a801672ca15ee82e09d473f1970..953de3b3927c80deb4e863b2b14cf032fd6e954e 100644 (file)
--- a/src/librbd/migration/OpenSourceImageRequest.cc
+++ b/src/librbd/migration/OpenSourceImageRequest.cc
@@ -132,11 +132,11 @@ void OpenSourceImageRequest<I>::open_format(
 
   // use default layout values (can be overridden by migration formats later)
   src_image_ctx->order = 22;
-  src_image_ctx->layout = file_layout_t();
-  src_image_ctx->layout.stripe_count = 1;
-  src_image_ctx->layout.stripe_unit = 1ULL << src_image_ctx->order;
-  src_image_ctx->layout.object_size = 1Ull << src_image_ctx->order;
-  src_image_ctx->layout.pool_id = -1;
+  src_image_ctx->stripe_unit = 1ULL << src_image_ctx->order;
+  src_image_ctx->stripe_count = 1;
+  src_image_ctx->layout = file_layout_t(src_image_ctx->stripe_unit,
+                                        src_image_ctx->stripe_count,
+                                        src_image_ctx->stripe_unit);
 
   SourceSpecBuilder<I> source_spec_builder{src_image_ctx};
   int r = source_spec_builder.build_format(source_spec_object, &m_format);