From 9f4f427df230e7241b3307e2cf164aade6c0d5df Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 2 Aug 2024 09:27:42 +0200 Subject: [PATCH] librbd/migration: make ImageDispatch handle encryption for non-native formats With NativeFormat now being handled via dispatch, handling encryption for non-native formats (i.e. mapping to raw image extents and performing decryption/mapping back on completion) in the migration layer is really straightforward. Note that alignment doesn't need to be performed in the migration layer because it happens on the destination image -- the "align and resubmit" logic in C_UnalignedObjectReadRequest should kick in before the call to read_parent(). Fixes: https://tracker.ceph.com/issues/53674 Co-authored-by: Or Ozeri Signed-off-by: Ilya Dryomov (cherry picked from commit 0000c3447407772039121bb4499f243df1c889da) Conflicts: qa/workunits/rbd/luks-encryption.sh [ - commit 9892ead7fcd9 ("librbd/crypto: allow loading luks format without specifying version") not in quincy -- pass through $format to test_migration_read_and_copyup() and test_migration_native_with_snaps() - commit 1d3de19c4005 ("tools/rbd: add encryption format support for cloned image") not in quincy -- drop test_migration_clone() ] src/librbd/migration/ImageDispatch.cc [ ImageArea support and commit 20aee5bbbcb5 ("neorados: Make IOContext getters/setters less weird") not in quincy ] --- qa/workunits/rbd/luks-encryption.sh | 114 +++++++++++++++++- src/librbd/migration/ImageDispatch.cc | 107 +++++++++++++++- .../migration/OpenSourceImageRequest.cc | 10 +- 3 files changed, 223 insertions(+), 8 deletions(-) diff --git a/qa/workunits/rbd/luks-encryption.sh b/qa/workunits/rbd/luks-encryption.sh index 49f7c6d8031ee..d0b0853e3cce4 100755 --- a/qa/workunits/rbd/luks-encryption.sh +++ b/qa/workunits/rbd/luks-encryption.sh @@ -2,7 +2,7 @@ set -ex CEPH_ID=${CEPH_ID:-admin} -TMP_FILES="/tmp/passphrase /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata" +TMP_FILES="/tmp/passphrase /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata /tmp/rawexport /tmp/export.qcow2" _sudo() { @@ -53,6 +53,103 @@ function test_encryption_format() { sudo cryptsetup close cryptsetupdev } +function test_migration_read_and_copyup() { + local format=$1 + + cp /tmp/testdata2 /tmp/cmpdata + + # test reading + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + + # trigger copyup at the beginning and at the end + xfs_io -c 'pwrite -S 0xab -W 0 4k' $LIBRBD_DEV /tmp/cmpdata + xfs_io -c 'pwrite -S 0xba -W 4095k 4k' $LIBRBD_DEV /tmp/cmpdata + + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is executed + rbd migration execute testimg1 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # test reading on a fresh mapping after migration is committed + rbd migration commit testimg1 + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/cmpdata + _sudo rbd device unmap -t nbd $LIBRBD_DEV +} + +function test_migration_native_with_snaps() { + local format=$1 + + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata2 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + test_migration_read_and_copyup $format + + # check that snapshots aren't affected by copyups + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap1 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1@snap2 -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + cmp $LIBRBD_DEV /tmp/testdata2 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + rbd snap rm testimg1@snap2 + rbd snap rm testimg1@snap1 + rbd rm testimg1 +} + +function test_migration() { + local format=$1 + + rbd encryption format testimg $format /tmp/passphrase + + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-format=$format,encryption-passphrase-file=/tmp/passphrase) + dd if=/tmp/testdata1 of=$LIBRBD_DEV conv=fsync bs=1M + rbd snap create testimg@snap1 + dd if=/tmp/testdata2 of=$LIBRBD_DEV conv=fsync bs=1M + rbd snap create testimg@snap2 + # FIXME: https://tracker.ceph.com/issues/67401 + # leave HEAD with the same data as snap2 as a workaround + # dd if=/tmp/testdata3 of=$LIBRBD_DEV conv=fsync bs=1M + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # live import a raw image + rbd export testimg /tmp/rawexport + rbd migration prepare --import-only --source-spec '{"type": "raw", "stream": {"type": "file", "file_path": "/tmp/rawexport"}}' testimg1 + test_migration_read_and_copyup $format + rbd rm testimg1 + + # live import a qcow image + qemu-img convert -f raw -O qcow2 /tmp/rawexport /tmp/export.qcow2 + rbd migration prepare --import-only --source-spec '{"type": "qcow", "stream": {"type": "file", "file_path": "/tmp/export.qcow2"}}' testimg1 + test_migration_read_and_copyup $format + rbd rm testimg1 + + # live import a native image + rbd migration prepare --import-only testimg@snap2 testimg1 + test_migration_native_with_snaps $format + + # live migrate a native image (removes testimg) + rbd migration prepare testimg testimg1 + test_migration_native_with_snaps $format + + rm /tmp/rawexport /tmp/export.qcow2 +} + function get_nbd_device_paths { rbd device list -t nbd | tail -n +2 | egrep "\s+rbd\s+testimg" | awk '{print $5;}' } @@ -67,7 +164,14 @@ function clean_up { for device in $(get_nbd_device_paths); do _sudo rbd device unmap -t nbd $device done - rbd ls | grep testimg > /dev/null && rbd rm testimg || true + + rbd migration abort testimg1 || true + rbd snap remove testimg1@snap2 || true + rbd snap remove testimg1@snap1 || true + rbd remove testimg1 || true + rbd snap remove testimg@snap2 || true + rbd snap remove testimg@snap1 || true + rbd remove testimg || true } if [[ $(uname) != "Linux" ]]; then @@ -104,4 +208,10 @@ test_encryption_format luks2 _sudo rbd device unmap -t nbd $RAW_DEV rbd rm testimg +rbd create --size 20M testimg +test_migration luks1 + +rbd create --size 32M testimg +test_migration luks2 + echo OK diff --git a/src/librbd/migration/ImageDispatch.cc b/src/librbd/migration/ImageDispatch.cc index e812f721c4f62..1fd1385707823 100644 --- a/src/librbd/migration/ImageDispatch.cc +++ b/src/librbd/migration/ImageDispatch.cc @@ -5,8 +5,13 @@ #include "include/neorados/RADOS.hpp" #include "common/dout.h" #include "librbd/ImageCtx.h" +#include "librbd/crypto/CryptoInterface.h" +#include "librbd/crypto/EncryptionFormat.h" #include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatcherInterface.h" +#include "librbd/io/Utils.h" #include "librbd/migration/FormatInterface.h" +#include "librbd/Utils.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -16,6 +21,72 @@ namespace librbd { namespace migration { +namespace { + +struct C_DecryptData : public io::ReadResult::C_ImageReadRequest { + crypto::CryptoInterface* crypto; + + C_DecryptData(io::AioCompletion* aio_comp, const io::Extents& image_extents, + crypto::CryptoInterface* crypto) + : C_ImageReadRequest(aio_comp, 0, image_extents), crypto(crypto) {} + + void finish(int r) override { + if (r < 0) { + C_ImageReadRequest::finish(r); + return; + } + + auto ciphertext_bl = std::move(bl); + for (const auto& extent : image_extents) { + ceph::bufferlist tmp; + ciphertext_bl.splice(0, extent.second, &tmp); + int r = crypto->decrypt(&tmp, extent.first); + if (r < 0) { + C_ImageReadRequest::finish(r); + return; + } + bl.claim_append(tmp); + } + + C_ImageReadRequest::finish(0); + } +}; + +template +struct C_MapSnapshotDelta : public io::C_AioRequest { + io::SnapshotDelta* snapshot_delta; + I* image_ctx; + + C_MapSnapshotDelta(io::AioCompletion* aio_comp, + io::SnapshotDelta* snapshot_delta, I* image_ctx) + : C_AioRequest(aio_comp), snapshot_delta(snapshot_delta), + image_ctx(image_ctx) {} + + void finish(int r) override { + if (r < 0) { + C_AioRequest::finish(r); + return; + } + + auto raw_snapshot_delta = std::move(*snapshot_delta); + for (const auto& [key, raw_sparse_extents] : raw_snapshot_delta) { + auto& sparse_extents = (*snapshot_delta)[key]; + for (const auto& raw_sparse_extent : raw_sparse_extents) { + io::Extents extents = {{raw_sparse_extent.get_off(), 0}}; + image_ctx->io_image_dispatcher->remap_extents( + extents, io::IMAGE_EXTENTS_MAP_TYPE_PHYSICAL_TO_LOGICAL); + sparse_extents.insert(extents[0].first, raw_sparse_extent.get_len(), + {raw_sparse_extent.get_val().state, + raw_sparse_extent.get_len()}); + } + } + + C_AioRequest::finish(0); + } +}; + +} // anonymous namespace + template ImageDispatch::ImageDispatch(I* image_ctx, std::unique_ptr format) @@ -50,6 +121,30 @@ bool ImageDispatch::read( } *dispatch_result = io::DISPATCH_RESULT_COMPLETE; + + if (m_image_ctx->crypto != nullptr) { + auto crypto = m_image_ctx->crypto; + + // alignment should be performed on the destination image (see + // C_UnalignedObjectReadRequest in CryptoObjectDispatch) + for (const auto& extent : image_extents) { + ceph_assert(crypto->is_aligned(extent.first, extent.second)); + } + + aio_comp->read_result = std::move(read_result); + aio_comp->read_result.set_image_extents(image_extents); + + aio_comp->set_request_count(1); + auto ctx = new C_DecryptData(aio_comp, image_extents, crypto); + aio_comp = io::AioCompletion::create_and_start( + ctx, util::get_image_ctx(m_image_ctx), io::AIO_TYPE_READ); + read_result = io::ReadResult(&ctx->bl); + + // map to raw image extents _after_ DATA area extents are captured + m_image_ctx->io_image_dispatcher->remap_extents( + image_extents, io::IMAGE_EXTENTS_MAP_TYPE_LOGICAL_TO_PHYSICAL); + } + m_format->read(aio_comp, io_context->read_snap().value_or(CEPH_NOSNAP), std::move(image_extents), std::move(read_result), op_flags, read_flags, parent_trace); @@ -148,7 +243,17 @@ bool ImageDispatch::list_snaps( *dispatch_result = io::DISPATCH_RESULT_COMPLETE; aio_comp->set_request_count(1); - auto ctx = new io::C_AioRequest(aio_comp); + Context* ctx; + + if (m_image_ctx->crypto != nullptr) { + // map to raw image extents + m_image_ctx->io_image_dispatcher->remap_extents( + image_extents, io::IMAGE_EXTENTS_MAP_TYPE_LOGICAL_TO_PHYSICAL); + // ... and back on completion + ctx = new C_MapSnapshotDelta(aio_comp, snapshot_delta, m_image_ctx); + } else { + ctx = new io::C_AioRequest(aio_comp); + } m_format->list_snaps(std::move(image_extents), std::move(snap_ids), list_snaps_flags, snapshot_delta, parent_trace, diff --git a/src/librbd/migration/OpenSourceImageRequest.cc b/src/librbd/migration/OpenSourceImageRequest.cc index b22900795b9d0..953de3b3927c8 100644 --- a/src/librbd/migration/OpenSourceImageRequest.cc +++ b/src/librbd/migration/OpenSourceImageRequest.cc @@ -132,11 +132,11 @@ void OpenSourceImageRequest::open_format( // use default layout values (can be overridden by migration formats later) src_image_ctx->order = 22; - src_image_ctx->layout = file_layout_t(); - src_image_ctx->layout.stripe_count = 1; - src_image_ctx->layout.stripe_unit = 1ULL << src_image_ctx->order; - src_image_ctx->layout.object_size = 1Ull << src_image_ctx->order; - src_image_ctx->layout.pool_id = -1; + src_image_ctx->stripe_unit = 1ULL << src_image_ctx->order; + src_image_ctx->stripe_count = 1; + src_image_ctx->layout = file_layout_t(src_image_ctx->stripe_unit, + src_image_ctx->stripe_count, + src_image_ctx->stripe_unit); SourceSpecBuilder source_spec_builder{src_image_ctx}; int r = source_spec_builder.build_format(source_spec_object, &m_format); -- 2.39.5