From 8bd3063197d21d145bd417b5d2074e2fe322eb3e Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 2 Aug 2024 09:27:42 +0200 Subject: [PATCH] librbd/migration: make ImageDispatch handle encryption for non-native formats With NativeFormat now being handled via dispatch, handling encryption for non-native formats (i.e. mapping to raw image extents and performing decryption/mapping back on completion) in the migration layer is really straightforward. Note that alignment doesn't need to be performed in the migration layer because it happens on the destination image -- the "align and resubmit" logic in C_UnalignedObjectReadRequest should kick in before the call to read_parent(). Fixes: https://tracker.ceph.com/issues/53674 Co-authored-by: Or Ozeri Signed-off-by: Ilya Dryomov (cherry picked from commit 0000c3447407772039121bb4499f243df1c889da) --- qa/workunits/rbd/luks-encryption.sh | 186 +++++++++++++++++- src/librbd/migration/ImageDispatch.cc | 112 ++++++++++- .../migration/OpenSourceImageRequest.cc | 10 +- 3 files changed, 301 insertions(+), 7 deletions(-) diff --git a/qa/workunits/rbd/luks-encryption.sh b/qa/workunits/rbd/luks-encryption.sh index c716a3de3a618..97cb5a0fe87ef 100755 --- a/qa/workunits/rbd/luks-encryption.sh +++ b/qa/workunits/rbd/luks-encryption.sh @@ -2,7 +2,7 @@ set -ex CEPH_ID=${CEPH_ID:-admin} -TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata" +TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata /tmp/rawexport /tmp/export.qcow2" _sudo() { @@ -154,6 +154,172 @@ function test_plaintext_detection { test_clone_and_load_with_a_single_passphrase false } +function test_migration_read_and_copyup() { + cp /tmp/testdata2 /tmp/cmpdata + + # test reading + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + + # trigger copyup at the beginning and at the end + xfs_io -c 'pwrite -S 0xab -W 0 4k' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xba -W 4095k 4k' $LIBRBD_DEV /tmp/cmpdata + + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is executed + rbd migration execute testimg1 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is committed + rbd migration commit testimg1 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV +} + +function test_migration_native_with_snaps() { + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata2 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + test_migration_read_and_copyup + + # check that snapshots aren't affected by copyups + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata2 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + rbd snap rm testimg1@snap2 + rbd snap rm testimg1@snap1 + rbd rm testimg1 +} + +function test_migration() { + local format=$1 + + rbd encryption format testimg $format /tmp/passphrase + + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase) + dd if=/tmp/testdata1 of=$LIBRBD_DEV conv=fsync bs=1M + rbd snap create testimg@snap1 + dd if=/tmp/testdata2 of=$LIBRBD_DEV conv=fsync bs=1M + rbd snap create testimg@snap2 + # FIXME: https://tracker.ceph.com/issues/67401 + # leave HEAD with the same data as snap2 as a workaround + # dd if=/tmp/testdata3 of=$LIBRBD_DEV conv=fsync bs=1M + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # live import a raw image + rbd export testimg /tmp/rawexport + rbd migration prepare --import-only --source-spec '{"type": "raw", "stream": {"type": "file", "file_path": "/tmp/rawexport"}}' testimg1 + test_migration_read_and_copyup + rbd rm testimg1 + + # live import a qcow image + qemu-img convert -f raw -O qcow2 /tmp/rawexport /tmp/export.qcow2 + rbd migration prepare --import-only --source-spec '{"type": "qcow", "stream": {"type": "file", "file_path": "/tmp/export.qcow2"}}' testimg1 + test_migration_read_and_copyup + rbd rm testimg1 + + # live import a native image + rbd migration prepare --import-only testimg@snap2 testimg1 + test_migration_native_with_snaps + + # live migrate a native image (removes testimg) + rbd migration prepare testimg testimg1 + test_migration_native_with_snaps + + rm /tmp/rawexport /tmp/export.qcow2 +} + +function test_migration_clone() { + local format=$1 + + truncate -s 0 /tmp/cmpdata + truncate -s 32M /tmp/cmpdata + + rbd encryption format testimg $format /tmp/passphrase + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase) + xfs_io -c 'pwrite -S 0xaa -W 4M 1M' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xaa -W 14M 1M' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xaa -W 25M 1M' $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + rbd snap create testimg@snap + rbd snap protect testimg@snap + rbd clone testimg@snap testimg1 + + rbd encryption format testimg1 $format /tmp/passphrase2 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase) + xfs_io -c 'pwrite -S 0xbb -W 2M 1M' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xbb -W 19M 1M' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xbb -W 28M 1M' $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # FIXME: https://tracker.ceph.com/issues/67402 + rbd config image set testimg1 rbd_sparse_read_threshold_bytes 1 + + # live migrate a native clone image (removes testimg1) + rbd migration prepare testimg1 testimg2 + + # test reading + # FIXME: https://tracker.ceph.com/issues/63184 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + + # trigger copyup for an unwritten area + xfs_io -c 'pwrite -S 0xcc -W 24167k 4k' $LIBRBD_DEV /tmp/cmpdata + + # trigger copyup for areas written in testimg (parent) + xfs_io -c 'pwrite -S 0xcc -W 4245k 4k' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xcc -W 13320k 4k' $LIBRBD_DEV /tmp/cmpdata + + # trigger copyup for areas written in testimg1 (clone) + xfs_io -c 'pwrite -S 0xcc -W 2084k 4k' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xcc -W 32612k 4k' $LIBRBD_DEV /tmp/cmpdata + + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping + # FIXME: https://tracker.ceph.com/issues/63184 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is executed + rbd migration execute testimg2 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is committed + rbd migration commit testimg2 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-passphrase-file=/tmp/passphrase2,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + rbd rm testimg2 + rbd snap unprotect testimg@snap + rbd snap rm testimg@snap + rbd rm testimg +} + function get_nbd_device_paths { rbd device list -t nbd | tail -n +2 | egrep "\s+rbd\s+testimg" | awk '{print $5;}' } @@ -169,10 +335,16 @@ function clean_up { _sudo rbd device unmap -t nbd $device done + rbd migration abort testimg2 || true rbd remove testimg2 || true + rbd migration abort testimg1 || true + rbd snap remove testimg1@snap2 || true + rbd snap remove testimg1@snap1 || true rbd snap unprotect testimg1@snap || true rbd snap remove testimg1@snap || true rbd remove testimg1 || true + rbd snap remove testimg@snap2 || true + rbd snap remove testimg@snap1 || true rbd snap unprotect testimg@snap || true rbd snap remove testimg@snap || true rbd remove testimg || true @@ -217,4 +389,16 @@ test_clone_encryption _sudo rbd device unmap -t nbd $RAW_DEV rbd rm testimg +rbd create --size 20M testimg +test_migration luks1 + +rbd create --size 32M testimg +test_migration luks2 + +rbd create --size 36M testimg +test_migration_clone luks1 + +rbd create --size 48M testimg +test_migration_clone luks2 + echo OK diff --git a/src/librbd/migration/ImageDispatch.cc b/src/librbd/migration/ImageDispatch.cc index 20cc3cb07e7eb..8420d06d1960b 100644 --- a/src/librbd/migration/ImageDispatch.cc +++ b/src/librbd/migration/ImageDispatch.cc @@ -5,8 +5,12 @@ #include "include/neorados/RADOS.hpp" #include "common/dout.h" #include "librbd/ImageCtx.h" +#include "librbd/crypto/CryptoInterface.h" +#include "librbd/crypto/EncryptionFormat.h" #include "librbd/io/AioCompletion.h" +#include "librbd/io/Utils.h" #include "librbd/migration/FormatInterface.h" +#include "librbd/Utils.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -16,6 +20,72 @@ namespace librbd { namespace migration { +namespace { + +struct C_DecryptData : public io::ReadResult::C_ImageReadRequest { + crypto::CryptoInterface* crypto; + + C_DecryptData(io::AioCompletion* aio_comp, const io::Extents& image_extents, + crypto::CryptoInterface* crypto) + : C_ImageReadRequest(aio_comp, 0, image_extents), crypto(crypto) {} + + void finish(int r) override { + if (r < 0) { + C_ImageReadRequest::finish(r); + return; + } + + auto ciphertext_bl = std::move(bl); + for (const auto& extent : image_extents) { + ceph::bufferlist tmp; + ciphertext_bl.splice(0, extent.second, &tmp); + int r = crypto->decrypt(&tmp, extent.first); + if (r < 0) { + C_ImageReadRequest::finish(r); + return; + } + bl.claim_append(tmp); + } + + C_ImageReadRequest::finish(0); + } +}; + +template +struct C_MapSnapshotDelta : public io::C_AioRequest { + io::SnapshotDelta* snapshot_delta; + I* image_ctx; + + C_MapSnapshotDelta(io::AioCompletion* aio_comp, + io::SnapshotDelta* snapshot_delta, I* image_ctx) + : C_AioRequest(aio_comp), snapshot_delta(snapshot_delta), + image_ctx(image_ctx) {} + + void finish(int r) override { + if (r < 0) { + C_AioRequest::finish(r); + return; + } + + auto raw_snapshot_delta = std::move(*snapshot_delta); + for (const auto& [key, raw_sparse_extents] : raw_snapshot_delta) { + auto& sparse_extents = (*snapshot_delta)[key]; + for (const auto& raw_sparse_extent : raw_sparse_extents) { + auto off = io::util::raw_to_area_offset(*image_ctx, + raw_sparse_extent.get_off()); + ceph_assert(off.second == io::ImageArea::DATA); + sparse_extents.insert(off.first, raw_sparse_extent.get_len(), + {raw_sparse_extent.get_val().state, + raw_sparse_extent.get_len()}); + } + } + + C_AioRequest::finish(0); + } +}; + +} // anonymous namespace + template ImageDispatch::ImageDispatch(I* image_ctx, std::unique_ptr format) @@ -50,6 +120,33 @@ bool ImageDispatch::read( } *dispatch_result = io::DISPATCH_RESULT_COMPLETE; + + if (m_image_ctx->encryption_format != nullptr && + (*image_dispatch_flags & io::IMAGE_DISPATCH_FLAG_CRYPTO_HEADER) == 0) { + auto crypto = m_image_ctx->encryption_format->get_crypto(); + + // alignment should be performed on the destination image (see + // C_UnalignedObjectReadRequest in CryptoObjectDispatch) + for (const auto& extent : image_extents) { + ceph_assert(crypto->is_aligned(extent.first, extent.second)); + } + + aio_comp->read_result = std::move(read_result); + aio_comp->read_result.set_image_extents(image_extents); + + aio_comp->set_request_count(1); + auto ctx = new C_DecryptData(aio_comp, image_extents, crypto); + aio_comp = io::AioCompletion::create_and_start( + ctx, util::get_image_ctx(m_image_ctx), io::AIO_TYPE_READ); + read_result = io::ReadResult(&ctx->bl); + + // map to raw image extents _after_ DATA area extents are captured + for (auto& extent : image_extents) { + extent.first = io::util::area_to_raw_offset(*m_image_ctx, extent.first, + io::ImageArea::DATA); + } + } + m_format->read(aio_comp, io_context->get_read_snap(), std::move(image_extents), std::move(read_result), op_flags, read_flags, parent_trace); @@ -148,7 +245,20 @@ bool ImageDispatch::list_snaps( *dispatch_result = io::DISPATCH_RESULT_COMPLETE; aio_comp->set_request_count(1); - auto ctx = new io::C_AioRequest(aio_comp); + Context* ctx; + + if (m_image_ctx->encryption_format != nullptr && + (*image_dispatch_flags & io::IMAGE_DISPATCH_FLAG_CRYPTO_HEADER) == 0) { + // map to raw image extents + for (auto& extent : image_extents) { + extent.first = io::util::area_to_raw_offset(*m_image_ctx, extent.first, + io::ImageArea::DATA); + } + // ... and back on completion + ctx = new C_MapSnapshotDelta(aio_comp, snapshot_delta, m_image_ctx); + } else { + ctx = new io::C_AioRequest(aio_comp); + } m_format->list_snaps(std::move(image_extents), std::move(snap_ids), list_snaps_flags, snapshot_delta, parent_trace, diff --git a/src/librbd/migration/OpenSourceImageRequest.cc b/src/librbd/migration/OpenSourceImageRequest.cc index b22900795b9d0..953de3b3927c8 100644 --- a/src/librbd/migration/OpenSourceImageRequest.cc +++ b/src/librbd/migration/OpenSourceImageRequest.cc @@ -132,11 +132,11 @@ void OpenSourceImageRequest::open_format( // use default layout values (can be overridden by migration formats later) src_image_ctx->order = 22; - src_image_ctx->layout = file_layout_t(); - src_image_ctx->layout.stripe_count = 1; - src_image_ctx->layout.stripe_unit = 1ULL << src_image_ctx->order; - src_image_ctx->layout.object_size = 1Ull << src_image_ctx->order; - src_image_ctx->layout.pool_id = -1; + src_image_ctx->stripe_unit = 1ULL << src_image_ctx->order; + src_image_ctx->stripe_count = 1; + src_image_ctx->layout = file_layout_t(src_image_ctx->stripe_unit, + src_image_ctx->stripe_count, + src_image_ctx->stripe_unit); SourceSpecBuilder source_spec_builder{src_image_ctx}; int r = source_spec_builder.build_format(source_spec_object, &m_format); -- 2.39.5