From 83ab7ae9dc8e9812a70378c4493f6851af2251d8 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 16 Aug 2024 19:09:39 +0200 Subject: [PATCH] librbd/migration: add external clusters support This commit extends NativeFormat (aka migration where the migration source is an RBD image) to support external Ceph clusters, limited to import-only mode. Co-authored-by: Or Ozeri Signed-off-by: Ilya Dryomov (cherry picked from commit 293d523ea69aaf1fa5c372f058f4426d49a3e196) --- doc/rbd/rbd-live-migration.rst | 19 ++-- qa/suites/rbd/migration-external/% | 0 qa/suites/rbd/migration-external/.qa | 1 + qa/suites/rbd/migration-external/1-base/.qa | 1 + .../migration-external/1-base/install.yaml | 8 ++ .../rbd/migration-external/2-clusters/.qa | 1 + .../migration-external/2-clusters/2-node.yaml | 15 ++++ .../rbd/migration-external/3-objectstore | 1 + .../4-supported-random-distro$ | 1 + .../rbd/migration-external/5-data-pool/.qa | 1 + .../migration-external/5-data-pool/ec.yaml | 29 ++++++ .../migration-external/5-data-pool/none.yaml | 0 .../5-data-pool/replicated.yaml | 14 +++ .../rbd/migration-external/6-prepare/.qa | 1 + .../6-prepare/native-clone.yaml | 29 ++++++ .../6-prepare/native-standalone.yaml | 18 ++++ .../rbd/migration-external/7-io-workloads/.qa | 1 + .../7-io-workloads/qemu_xfstests.yaml | 14 +++ .../8-migrate-workloads/.qa | 1 + .../8-migrate-workloads/execute.yaml | 14 +++ qa/suites/rbd/migration-external/conf | 1 + src/librbd/ImageCtx.cc | 1 + src/librbd/ImageCtx.h | 4 +- src/librbd/api/Migration.cc | 9 +- src/librbd/image/CloseRequest.cc | 5 ++ src/librbd/image/RefreshParentRequest.cc | 8 +- src/librbd/image/RefreshParentRequest.h | 2 + src/librbd/migration/NativeFormat.cc | 89 ++++++++++++++++++- src/librbd/migration/NativeFormat.h | 3 +- .../migration/OpenSourceImageRequest.cc | 24 ++++- src/librbd/migration/OpenSourceImageRequest.h | 9 +- 31 files changed, 302 insertions(+), 22 deletions(-) create mode 100644 qa/suites/rbd/migration-external/% create mode 120000 qa/suites/rbd/migration-external/.qa create mode 120000 qa/suites/rbd/migration-external/1-base/.qa create mode 100644 qa/suites/rbd/migration-external/1-base/install.yaml create mode 120000 qa/suites/rbd/migration-external/2-clusters/.qa create mode 100644 qa/suites/rbd/migration-external/2-clusters/2-node.yaml create mode 120000 qa/suites/rbd/migration-external/3-objectstore create mode 120000 qa/suites/rbd/migration-external/4-supported-random-distro$ create mode 120000 qa/suites/rbd/migration-external/5-data-pool/.qa create mode 100644 qa/suites/rbd/migration-external/5-data-pool/ec.yaml create mode 100644 qa/suites/rbd/migration-external/5-data-pool/none.yaml create mode 100644 qa/suites/rbd/migration-external/5-data-pool/replicated.yaml create mode 120000 qa/suites/rbd/migration-external/6-prepare/.qa create mode 100644 qa/suites/rbd/migration-external/6-prepare/native-clone.yaml create mode 100644 qa/suites/rbd/migration-external/6-prepare/native-standalone.yaml create mode 120000 qa/suites/rbd/migration-external/7-io-workloads/.qa create mode 100644 qa/suites/rbd/migration-external/7-io-workloads/qemu_xfstests.yaml create mode 120000 qa/suites/rbd/migration-external/8-migrate-workloads/.qa create mode 100644 qa/suites/rbd/migration-external/8-migrate-workloads/execute.yaml create mode 120000 qa/suites/rbd/migration-external/conf diff --git a/doc/rbd/rbd-live-migration.rst b/doc/rbd/rbd-live-migration.rst index 6e0e635b2d12f..68165c18ae647 100644 --- a/doc/rbd/rbd-live-migration.rst +++ b/doc/rbd/rbd-live-migration.rst @@ -4,11 +4,11 @@ .. index:: Ceph Block Device; live-migration -RBD images can be live-migrated between different pools within the same cluster; -between different image formats and layouts; or from external data sources. -When started, the source will be deep-copied to the destination image, pulling -all snapshot history while preserving the sparse allocation of data where -possible. +RBD images can be live-migrated between different pools, image formats and/or +layouts within the same Ceph cluster; from an image in another Ceph cluster; or +from external data sources. When started, the source will be deep-copied to +the destination image, pulling all snapshot history while preserving the sparse +allocation of data where possible. By default, when live-migrating RBD images within the same Ceph cluster, the source image will be marked read-only and all clients will instead redirect @@ -18,8 +18,9 @@ image during the migration to remove the dependency on the source image's parent. The live-migration process can also be used in an import-only mode where the -source image remains unmodified and the target image can be linked to an -external data source such as a backing file, HTTP(s) file, or S3 object. +source image remains unmodified and the target image can be linked to an image +in another Ceph cluster or to an external data source such as a backing file, +HTTP(s) file, or S3 object. The live-migration copy process can safely run in the background while the new target image is in use. There is currently a requirement to temporarily stop @@ -156,6 +157,10 @@ as follows:: { "type": "native", + ["cluster_name": "",] (specify if image in another cluster, + requires ``.conf`` file) + ["client_name": "",] (for connecting to another cluster, + default is ``client.admin``) "pool_name": "", ["pool_id": ,] (optional alternative to "pool_name") ["pool_namespace": "::prepare_import( << dest_io_ctx.get_pool_name() << "/" << dest_image_name << ", opts=" << opts << dendl; - I* src_image_ctx = nullptr; + I* src_image_ctx; + librados::Rados* src_rados; C_SaferCond open_ctx; auto req = migration::OpenSourceImageRequest::create( dest_io_ctx, nullptr, CEPH_NOSNAP, - {-1, "", "", "", source_spec, {}, 0, false}, &src_image_ctx, &open_ctx); + {-1, "", "", "", source_spec, {}, 0, false}, &src_image_ctx, &src_rados, + &open_ctx); req->send(); int r = open_ctx.wait(); @@ -540,8 +542,9 @@ int Migration::prepare_import( return r; } - BOOST_SCOPE_EXIT_TPL(src_image_ctx) { + BOOST_SCOPE_EXIT_TPL(src_image_ctx, src_rados) { src_image_ctx->state->close(); + delete src_rados; } BOOST_SCOPE_EXIT_END; uint64_t image_format = 2; diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc index 7293687f5b81c..eac755e457949 100644 --- a/src/librbd/image/CloseRequest.cc +++ b/src/librbd/image/CloseRequest.cc @@ -307,6 +307,11 @@ void CloseRequest::handle_close_parent(int r) { ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; m_image_ctx->parent = nullptr; + if (m_image_ctx->parent_rados != nullptr) { + delete m_image_ctx->parent_rados; + m_image_ctx->parent_rados = nullptr; + } + save_result(r); if (r < 0) { lderr(cct) << "error closing parent image: " << cpp_strerror(r) << dendl; diff --git a/src/librbd/image/RefreshParentRequest.cc b/src/librbd/image/RefreshParentRequest.cc index bfaef2dcc2f42..75bf64ee9c052 100644 --- a/src/librbd/image/RefreshParentRequest.cc +++ b/src/librbd/image/RefreshParentRequest.cc @@ -90,6 +90,7 @@ template void RefreshParentRequest::apply() { ceph_assert(ceph_mutex_is_wlocked(m_child_image_ctx.image_lock)); std::swap(m_child_image_ctx.parent, m_parent_image_ctx); + std::swap(m_child_image_ctx.parent_rados, m_parent_rados); } template @@ -101,6 +102,7 @@ void RefreshParentRequest::finalize(Context *on_finish) { if (m_parent_image_ctx != nullptr) { send_close_parent(); } else { + ceph_assert(m_parent_rados == nullptr); send_complete(0); } } @@ -119,7 +121,7 @@ void RefreshParentRequest::send_open_parent() { &RefreshParentRequest::handle_open_parent, false>(this)); auto req = migration::OpenSourceImageRequest::create( m_child_image_ctx.md_ctx, &m_child_image_ctx, m_parent_md.spec.snap_id, - m_migration_info, &m_parent_image_ctx, ctx); + m_migration_info, &m_parent_image_ctx, &m_parent_rados, ctx); req->send(); return; } @@ -188,6 +190,10 @@ Context *RefreshParentRequest::handle_close_parent(int *result) { ldout(cct, 10) << this << " " << __func__ << " r=" << *result << dendl; m_parent_image_ctx = nullptr; + if (m_parent_rados != nullptr) { + delete m_parent_rados; + m_parent_rados = nullptr; + } if (*result < 0) { lderr(cct) << "failed to close parent image: " << cpp_strerror(*result) diff --git a/src/librbd/image/RefreshParentRequest.h b/src/librbd/image/RefreshParentRequest.h index 100fa4fd1121f..271e856418dc3 100644 --- a/src/librbd/image/RefreshParentRequest.h +++ b/src/librbd/image/RefreshParentRequest.h @@ -5,6 +5,7 @@ #define CEPH_LIBRBD_IMAGE_REFRESH_PARENT_REQUEST_H #include "include/int_types.h" +#include "include/rados/librados_fwd.hpp" #include "librbd/Types.h" class Context; @@ -68,6 +69,7 @@ private: Context *m_on_finish; ImageCtxT *m_parent_image_ctx = nullptr; + librados::Rados *m_parent_rados = nullptr; int m_error_result; diff --git a/src/librbd/migration/NativeFormat.cc b/src/librbd/migration/NativeFormat.cc index d9c030f524c90..4bbeee0535c6a 100644 --- a/src/librbd/migration/NativeFormat.cc +++ b/src/librbd/migration/NativeFormat.cc @@ -2,8 +2,11 @@ // vim: ts=8 sw=2 smarttab #include "librbd/migration/NativeFormat.h" +#include "common/ceph_argparse.h" +#include "common/common_init.h" #include "common/dout.h" #include "common/errno.h" +#include "include/scope_guard.h" #include "librbd/ImageCtx.h" #include "json_spirit/json_spirit.h" #include "boost/lexical_cast.hpp" @@ -19,6 +22,8 @@ namespace migration { namespace { const std::string TYPE_KEY{"type"}; +const std::string CLUSTER_NAME_KEY{"cluster_name"}; +const std::string CLIENT_NAME_KEY{"client_name"}; const std::string POOL_ID_KEY{"pool_id"}; const std::string POOL_NAME_KEY{"pool_name"}; const std::string POOL_NAMESPACE_KEY{"pool_namespace"}; @@ -57,8 +62,11 @@ template int NativeFormat::create_image_ctx( librados::IoCtx& dst_io_ctx, const json_spirit::mObject& source_spec_object, - bool import_only, uint64_t src_snap_id, I** src_image_ctx) { + bool import_only, uint64_t src_snap_id, I** src_image_ctx, + librados::Rados** src_rados) { auto cct = reinterpret_cast(dst_io_ctx.cct()); + std::string cluster_name; + std::string client_name; std::string pool_name; int64_t pool_id = -1; std::string pool_namespace; @@ -68,6 +76,30 @@ int NativeFormat::create_image_ctx( uint64_t snap_id = CEPH_NOSNAP; int r; + if (auto it = source_spec_object.find(CLUSTER_NAME_KEY); + it != source_spec_object.end()) { + if (it->second.type() == json_spirit::str_type) { + cluster_name = it->second.get_str(); + } else { + lderr(cct) << "invalid cluster name" << dendl; + return -EINVAL; + } + } + + if (auto it = source_spec_object.find(CLIENT_NAME_KEY); + it != source_spec_object.end()) { + if (cluster_name.empty()) { + lderr(cct) << "cannot specify client name without cluster name" << dendl; + return -EINVAL; + } + if (it->second.type() == json_spirit::str_type) { + client_name = it->second.get_str(); + } else { + lderr(cct) << "invalid client name" << dendl; + return -EINVAL; + } + } + if (auto it = source_spec_object.find(POOL_NAME_KEY); it != source_spec_object.end()) { if (it->second.type() == json_spirit::str_type) { @@ -179,7 +211,53 @@ int NativeFormat::create_image_ctx( snap_id = src_snap_id; } - // TODO add support for external clusters + std::unique_ptr rados_ptr; + if (!cluster_name.empty()) { + // manually bootstrap a CephContext, skipping reading environment + // variables for now -- since we don't have access to command line + // arguments here, the least confusing option is to limit initial + // remote cluster config to a file in the default location + // TODO: support specifying mon_host and key via source spec + // TODO: support merging in effective local cluster config to get + // overrides for log levels, etc + CephInitParameters iparams(CEPH_ENTITY_TYPE_CLIENT); + if (!client_name.empty() && !iparams.name.from_str(client_name)) { + lderr(cct) << "failed to set remote client name" << dendl; + return -EINVAL; + } + + auto remote_cct = common_preinit(iparams, CODE_ENVIRONMENT_LIBRARY, 0); + auto put_remote_cct = make_scope_guard([remote_cct] { remote_cct->put(); }); + + remote_cct->_conf->cluster = cluster_name; + + // pass CEPH_CONF_FILE_DEFAULT instead of nullptr to prevent + // CEPH_CONF environment variable from being picked up + r = remote_cct->_conf.parse_config_files(CEPH_CONF_FILE_DEFAULT, nullptr, + 0); + if (r < 0) { + remote_cct->_conf.complain_about_parse_error(cct); + lderr(cct) << "failed to read ceph conf for remote cluster: " + << cpp_strerror(r) << dendl; + return r; + } + + remote_cct->_conf.apply_changes(nullptr); + + rados_ptr.reset(new librados::Rados()); + r = rados_ptr->init_with_context(remote_cct); + ceph_assert(r == 0); + + r = rados_ptr->connect(); + if (r < 0) { + lderr(cct) << "failed to connect to remote cluster: " << cpp_strerror(r) + << dendl; + return r; + } + } else { + rados_ptr.reset(new librados::Rados(dst_io_ctx)); + } + librados::IoCtx src_io_ctx; if (!pool_name.empty()) { r = rados_ptr->ioctx_create(pool_name.c_str(), src_io_ctx); @@ -201,6 +279,13 @@ int NativeFormat::create_image_ctx( *src_image_ctx = I::create(image_name, image_id, snap_id, src_io_ctx, true); } + + if (!cluster_name.empty()) { + *src_rados = rados_ptr.release(); + } else { + *src_rados = nullptr; + } + return 0; } diff --git a/src/librbd/migration/NativeFormat.h b/src/librbd/migration/NativeFormat.h index b4a84ae3e1838..581c6c0bb2d6a 100644 --- a/src/librbd/migration/NativeFormat.h +++ b/src/librbd/migration/NativeFormat.h @@ -28,7 +28,8 @@ public: static int create_image_ctx(librados::IoCtx& dst_io_ctx, const json_spirit::mObject& source_spec_object, bool import_only, uint64_t src_snap_id, - ImageCtxT** src_image_ctx); + ImageCtxT** src_image_ctx, + librados::Rados** src_rados); }; } // namespace migration diff --git a/src/librbd/migration/OpenSourceImageRequest.cc b/src/librbd/migration/OpenSourceImageRequest.cc index 953de3b3927c8..2bd2d1935f49a 100644 --- a/src/librbd/migration/OpenSourceImageRequest.cc +++ b/src/librbd/migration/OpenSourceImageRequest.cc @@ -6,6 +6,7 @@ #include "common/errno.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" +#include "librbd/TaskFinisher.h" #include "librbd/Utils.h" #include "librbd/io/ImageDispatcher.h" #include "librbd/migration/FormatInterface.h" @@ -24,11 +25,13 @@ namespace migration { template OpenSourceImageRequest::OpenSourceImageRequest( librados::IoCtx& dst_io_ctx, I* dst_image_ctx, uint64_t src_snap_id, - const MigrationInfo &migration_info, I** src_image_ctx, Context* on_finish) + const MigrationInfo &migration_info, I** src_image_ctx, + librados::Rados** src_rados, Context* on_finish) : m_cct(reinterpret_cast(dst_io_ctx.cct())), m_dst_io_ctx(dst_io_ctx), m_dst_image_ctx(dst_image_ctx), m_src_snap_id(src_snap_id), m_migration_info(migration_info), - m_src_image_ctx(src_image_ctx), m_on_finish(on_finish) { + m_src_image_ctx(src_image_ctx), m_src_rados(src_rados), + m_on_finish(on_finish) { ldout(m_cct, 10) << dendl; } @@ -74,7 +77,7 @@ void OpenSourceImageRequest::open_native( int r = NativeFormat::create_image_ctx(m_dst_io_ctx, source_spec_object, import_only, m_src_snap_id, - m_src_image_ctx); + m_src_image_ctx, m_src_rados); if (r < 0) { lderr(m_cct) << "failed to create native image context: " << cpp_strerror(r) << dendl; @@ -113,7 +116,17 @@ void OpenSourceImageRequest::handle_open_native(int r) { if (r < 0) { lderr(m_cct) << "failed to open native image: " << cpp_strerror(r) << dendl; - finish(r); + + // m_src_rados must be deleted outside the scope of its task + // finisher thread to avoid the finisher attempting to destroy + // itself and locking up + // since the local image (m_dst_image_ctx) may not be available, + // redirect to the local rados' task finisher + auto ctx = new LambdaContext([this](int r) { + delete *m_src_rados; + finish(r); + }); + TaskFinisherSingleton::get_singleton(m_cct).queue(ctx, r); return; } @@ -127,6 +140,8 @@ void OpenSourceImageRequest::open_format( // note that all source image ctx properties are placeholders *m_src_image_ctx = I::create("", "", CEPH_NOSNAP, m_dst_io_ctx, true); + *m_src_rados = nullptr; + auto src_image_ctx = *m_src_image_ctx; src_image_ctx->child = m_dst_image_ctx; @@ -293,6 +308,7 @@ void OpenSourceImageRequest::finish(int r) { if (r < 0) { *m_src_image_ctx = nullptr; + *m_src_rados = nullptr; } else { register_image_dispatch(); } diff --git a/src/librbd/migration/OpenSourceImageRequest.h b/src/librbd/migration/OpenSourceImageRequest.h index 465e0aa14f35a..a62c6a8d00695 100644 --- a/src/librbd/migration/OpenSourceImageRequest.h +++ b/src/librbd/migration/OpenSourceImageRequest.h @@ -27,18 +27,20 @@ public: ImageCtxT* destination_image_ctx, uint64_t src_snap_id, const MigrationInfo &migration_info, - ImageCtxT** source_image_ctx, + ImageCtxT** src_image_ctx, + librados::Rados** src_rados, Context* on_finish) { return new OpenSourceImageRequest(dst_io_ctx, destination_image_ctx, src_snap_id, migration_info, - source_image_ctx, on_finish); + src_image_ctx, src_rados, on_finish); } OpenSourceImageRequest(librados::IoCtx& dst_io_ctx, ImageCtxT* destination_image_ctx, uint64_t src_snap_id, const MigrationInfo &migration_info, - ImageCtxT** source_image_ctx, + ImageCtxT** src_image_ctx, + librados::Rados** src_rados, Context* on_finish); void send(); @@ -79,6 +81,7 @@ private: uint64_t m_src_snap_id; MigrationInfo m_migration_info; ImageCtxT** m_src_image_ctx; + librados::Rados** m_src_rados; Context* m_on_finish; std::unique_ptr m_format; -- 2.39.5