From eedfc17597ddde9fc83b74fc40ec70f9c33583d9 Mon Sep 17 00:00:00 2001 From: N Balachandran Date: Tue, 17 Dec 2024 15:10:11 +0530 Subject: [PATCH] rbd-mirror: default <-> non-default namespace remapping These changes allow the pool to be configured to disable mirroring on the default namespace if required (init-only mode). It also allows the default namespace to be mirrored to a non-default namespace on the remote pool. Co-authored-by: Ilya Dryomov Signed-off-by: N Balachandran Signed-off-by: Ilya Dryomov --- doc/rbd/rbd-mirroring.rst | 58 +++- qa/workunits/rbd/rbd_mirror.sh | 38 ++- src/cls/rbd/cls_rbd.cc | 4 +- src/cls/rbd/cls_rbd_types.cc | 3 + src/cls/rbd/cls_rbd_types.h | 7 +- src/include/rbd/librbd.h | 4 +- src/librbd/api/Migration.cc | 3 +- src/librbd/api/Mirror.cc | 28 +- .../dashboard/controllers/rbd_mirroring.py | 5 +- src/pybind/rbd/c_rbd.pxd | 1 + src/pybind/rbd/mock_rbd.pxi | 1 + src/pybind/rbd/rbd.pyx | 1 + src/test/cli/rbd/help.t | 2 +- src/test/cli/rbd/not-enough-args.t | 2 +- src/test/pybind/test_rbd.py | 88 ++++- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 303 ++++++++++++++---- src/tools/rbd/action/MirrorPool.cc | 39 ++- src/tools/rbd_mirror/NamespaceReplayer.cc | 6 + src/tools/rbd_mirror/PoolReplayer.cc | 106 ++---- src/tools/rbd_mirror/PoolReplayer.h | 1 - src/tools/rbd_mirror/ServiceDaemon.cc | 24 +- 21 files changed, 498 insertions(+), 226 deletions(-) diff --git a/doc/rbd/rbd-mirroring.rst b/doc/rbd/rbd-mirroring.rst index add0e9503b0ab..eb94a3716d548 100644 --- a/doc/rbd/rbd-mirroring.rst +++ b/doc/rbd/rbd-mirroring.rst @@ -87,12 +87,20 @@ site name to describe the local cluster:: rbd mirror pool enable [--site-name {local-site-name}] {pool-name} {mode} -The mirroring mode can either be ``image`` or ``pool``: - -* **image**: When configured in ``image`` mode, mirroring must - `explicitly enabled`_ on each image. -* **pool** (default): When configured in ``pool`` mode, all images in the pool - with the journaling feature enabled are mirrored. +The mirroring mode can be ``image``, ``pool`` or ``init-only``: + +* **image**: When configured in ``image`` mode, mirroring must be + `explicitly enabled`_ for each intended image in the default namespace + of the pool. Other namespaces aren't affected and must be + `configured separately`_. +* **pool**: When configured in ``pool`` mode, all images in the default + namespace of the pool with the journaling feature enabled are mirrored. + Other namespaces aren't affected and must be `configured separately`_. +* **init-only**: When configured in ``init-only`` mode, no images in the + default namespace of the pool will be mirrored but other namespaces can + still be configured. This is needed to allow some other namespace to be + mirrored to the default namespace of the remote pool but can be useful + on its own as well. For example:: @@ -237,9 +245,11 @@ pool as follows: Namespace Configuration ======================= -Mirroring can be configured on a namespace in a pool. The pool must already -have been configured for mirroring. The namespace can be mirrored to a namespace -with the same or a different name in the remote pool. +Mirroring can be enabled on non-default namespaces of a pool independent of +the default namespace. The pool must be configured for mirroring in advance. +A given namespace can be mirrored to a namespace with the same or a different +name in the remote pool, including to the default namespace (referred to as +``''`` or ``""``). Enable Mirroring ---------------- @@ -252,9 +262,9 @@ remote namespace name:: The mirroring mode can either be ``image`` or ``pool``: -* **image**: When configured in ``image`` mode, mirroring must - `explicitly enabled`_ on each image. -* **pool** (default): When configured in ``pool`` mode, all images in the namespace +* **image**: When configured in ``image`` mode, mirroring must be + `explicitly enabled`_ for each intended image in the namespace. +* **pool**: When configured in ``pool`` mode, all images in the namespace with the journaling feature enabled are mirrored. For example:: @@ -262,13 +272,30 @@ For example:: $ rbd --cluster site-a mirror pool enable image-pool/namespace-a image --remote-namespace namespace-b $ rbd --cluster site-b mirror pool enable image-pool/namespace-b image --remote-namespace namespace-a -This will set up image mode mirroring between image-pool/namespace-a on cluster -site-a and image-pool/namespace-b on cluster site-b. -The namespace and remote-namespace pair configured on a cluster must +This will set up image mode mirroring between ``image-pool/namespace-a`` on +cluster ``site-a`` and ``image-pool/namespace-b`` on cluster ``site-b``. +The namespace and remote-namespace pair configured on a local cluster must match the remote-namespace and namespace respectively on the remote cluster. If the ``--remote-namespace`` option is not provided, the namespace will be mirrored to a namespace with the same name in the remote pool. +To set up pool mode mirroring between ``image-pool`` (default namespace) on +cluster ``site-a`` and ``image-pool/namespace-c`` on cluster ``site-b``:: + + $ rbd --cluster site-a mirror pool enable image-pool pool --remote-namespace namespace-c + $ rbd --cluster site-b mirror pool enable image-pool init-only + $ rbd --cluster site-b mirror pool enable image-pool/namespace-c pool --remote-namespace "" + +To set up pool mode mirroring between ``image-pool`` (default namespace) on +cluster ``site-a`` and ``image-pool/namespace-d`` on cluster ``site-b`` and +at the same time image mode mirroring between ``image-pool`` (default namespace) +on cluster ``site-b`` and ``image-pool/namespace-e`` on cluster ``site-a``:: + + $ rbd --cluster site-a mirror pool enable image-pool pool --remote-namespace namespace-d + $ rbd --cluster site-a mirror pool enable image-pool/namespace-e image --remote-namespace "" + $ rbd --cluster site-b mirror pool enable image-pool image --remote-namespace namespace-e + $ rbd --cluster site-b mirror pool enable image-pool/namespace-d pool --remote-namespace "" + Disable Mirroring ----------------- @@ -584,6 +611,7 @@ The ``rbd-mirror`` can also be run in foreground by ``rbd-mirror`` command:: .. _rbd: ../../man/8/rbd .. _ceph-conf: ../../rados/configuration/ceph-conf/#running-multiple-clusters .. _explicitly enabled: #enable-image-mirroring +.. _configured separately: #namespace-configuration .. _bootstrap token: #bootstrap-peers .. _force resync command: #force-image-resync .. _demote the image: #image-promotion-and-demotion diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 90d5204b92feb..e57ef382e6c5d 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -29,9 +29,45 @@ fi . $(dirname $0)/rbd_mirror_helpers.sh setup -testlog "TEST: add image and test replay" +testlog "TEST: mirror from default namespace to non-default namespace" start_mirrors ${CLUSTER1} image=test +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} init-only +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} init-only +rbd --cluster ${CLUSTER1} mirror pool disable ${POOL}/${NS1} +rbd --cluster ${CLUSTER2} mirror pool disable ${POOL}/${NS1} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} ${MIRROR_POOL_MODE} --remote-namespace ${NS1} +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} --remote-namespace "" +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL} ${image} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL} ${image} +remove_image_retry ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' +rbd --cluster ${CLUSTER1} mirror pool disable ${POOL}/${NS1} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} init-only + +testlog "TEST: mirror from non-default namespace to default namespace" +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} --remote-namespace "" +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} ${MIRROR_POOL_MODE} --remote-namespace ${NS1} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} ${RBD_MIRROR_MODE} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL}/${NS1} ${image} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL}/${NS1} ${image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} init-only +rbd --cluster ${CLUSTER2} mirror pool disable ${POOL}/${NS1} +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} ${MIRROR_POOL_MODE} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} ${MIRROR_POOL_MODE} +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} + +testlog "TEST: add image and test replay" create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index 8489f0ca818ff..b6e2af50064ef 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -5877,6 +5877,7 @@ int mirror_mode_set(cls_method_context_t hctx, bufferlist *in, break; case cls::rbd::MIRROR_MODE_IMAGE: case cls::rbd::MIRROR_MODE_POOL: + case cls::rbd::MIRROR_MODE_INIT_ONLY: enabled = true; break; default: @@ -5960,7 +5961,8 @@ int mirror_remote_namespace_set(cls_method_context_t hctx, bufferlist *in, int r = read_key(hctx, mirror::MODE, &mirror_mode); if (r < 0 && r != -ENOENT) { return r; - } else if (r == 0 && mirror_mode != cls::rbd::MIRROR_MODE_DISABLED) { + } else if (r == 0 && (mirror_mode != cls::rbd::MIRROR_MODE_DISABLED && + mirror_mode != cls::rbd::MIRROR_MODE_INIT_ONLY)) { CLS_ERR("cannot set mirror remote namespace while mirroring enabled"); return -EINVAL; } diff --git a/src/cls/rbd/cls_rbd_types.cc b/src/cls/rbd/cls_rbd_types.cc index b6177e85eb84d..1fdfbd0f63e0c 100644 --- a/src/cls/rbd/cls_rbd_types.cc +++ b/src/cls/rbd/cls_rbd_types.cc @@ -109,6 +109,9 @@ std::ostream& operator<<(std::ostream& os, const MirrorMode& mirror_mode) { case MIRROR_MODE_POOL: os << "pool"; break; + case MIRROR_MODE_INIT_ONLY: + os << "init-only"; + break; default: os << "unknown (" << static_cast(mirror_mode) << ")"; break; diff --git a/src/cls/rbd/cls_rbd_types.h b/src/cls/rbd/cls_rbd_types.h index 00de0a1e4c784..d5f65abd75f44 100644 --- a/src/cls/rbd/cls_rbd_types.h +++ b/src/cls/rbd/cls_rbd_types.h @@ -46,9 +46,10 @@ inline void decode(DirectoryState &state, ceph::buffer::list::const_iterator& it } enum MirrorMode { - MIRROR_MODE_DISABLED = 0, - MIRROR_MODE_IMAGE = 1, - MIRROR_MODE_POOL = 2 + MIRROR_MODE_DISABLED = 0, + MIRROR_MODE_IMAGE = 1, + MIRROR_MODE_POOL = 2, + MIRROR_MODE_INIT_ONLY = 3 }; enum GroupImageLinkState { diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 9a5d539fd0548..fc270b9ab3f38 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -151,7 +151,9 @@ typedef struct { typedef enum { RBD_MIRROR_MODE_DISABLED, /* mirroring is disabled */ RBD_MIRROR_MODE_IMAGE, /* mirroring enabled on a per-image basis */ - RBD_MIRROR_MODE_POOL /* mirroring enabled on all journaled images */ + RBD_MIRROR_MODE_POOL, /* mirroring enabled on all journaled images */ + RBD_MIRROR_MODE_INIT_ONLY /* mirroring is initialized but not enabled + (default namespace only) */ } rbd_mirror_mode_t; typedef enum { diff --git a/src/librbd/api/Migration.cc b/src/librbd/api/Migration.cc index a166a4c5752ac..4ae95f692fb5d 100644 --- a/src/librbd/api/Migration.cc +++ b/src/librbd/api/Migration.cc @@ -1757,7 +1757,8 @@ int Migration::enable_mirroring( return r; } - if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED || + mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY) { ldout(m_cct, 10) << "mirroring is not enabled for destination pool" << dendl; return 0; diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index fa22084eb2ba5..910d3e4fba83f 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -444,7 +444,8 @@ int Mirror::image_enable(I *ictx, mirror_image_mode_t mode, return r; } - if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED || + mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY) { lderr(cct) << "cannot enable mirroring: mirroring is not enabled on a " << pool_or_namespace(ictx) << dendl; return -EINVAL; @@ -1065,6 +1066,7 @@ int Mirror::mode_get(librados::IoCtx& io_ctx, case cls::rbd::MIRROR_MODE_DISABLED: case cls::rbd::MIRROR_MODE_IMAGE: case cls::rbd::MIRROR_MODE_POOL: + case cls::rbd::MIRROR_MODE_INIT_ONLY: *mirror_mode = static_cast(mirror_mode_internal); break; default: @@ -1087,6 +1089,7 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, case RBD_MIRROR_MODE_DISABLED: case RBD_MIRROR_MODE_IMAGE: case RBD_MIRROR_MODE_POOL: + case RBD_MIRROR_MODE_INIT_ONLY: next_mirror_mode = static_cast(mirror_mode); break; default: @@ -1095,6 +1098,12 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, return -EINVAL; } + if (next_mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY && + !io_ctx.get_namespace().empty()) { + lderr(cct) << "init-only mode cannot be set on a namespace" << dendl; + return -EINVAL; + } + int r; if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { // fail early if pool still has peers registered and attempting to disable @@ -1192,7 +1201,8 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, } } } - } else if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + } else if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED || + next_mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY) { while (true) { bool retry_busy = false; bool pending_busy = false; @@ -1296,20 +1306,6 @@ int Mirror::remote_namespace_set(librados::IoCtx& io_ctx, const std::string& remote_namespace) { CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 20) << dendl; - - std::string local_namespace = io_ctx.get_namespace(); - - if (local_namespace.empty() && !remote_namespace.empty()) { - lderr(cct) << "cannot mirror the default namespace to a " - << "non-default namespace." << dendl; - return -EINVAL; - } - - if (!local_namespace.empty() && remote_namespace.empty()) { - lderr(cct) << "cannot mirror a non-default namespace to the default " - << "namespace." << dendl; - return -EINVAL; - } int r = cls_client::mirror_remote_namespace_set(&io_ctx, remote_namespace); if (r < 0) { diff --git a/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py b/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py index 1e80de74b3b94..413ccb8dde4c3 100644 --- a/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py +++ b/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py @@ -194,6 +194,8 @@ def _get_pool_stats(pool_names): mirror_mode = "image" elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL: mirror_mode = "pool" + elif mirror_mode == rbd.RBD_MIRROR_MODE_INIT_ONLY: + mirror_mode = "init-only" else: mirror_mode = "unknown" @@ -487,7 +489,8 @@ class RbdMirroringPoolMode(RESTController): MIRROR_MODES = { rbd.RBD_MIRROR_MODE_DISABLED: 'disabled', rbd.RBD_MIRROR_MODE_IMAGE: 'image', - rbd.RBD_MIRROR_MODE_POOL: 'pool' + rbd.RBD_MIRROR_MODE_POOL: 'pool', + rbd.RBD_MIRROR_MODE_INIT_ONLY: 'init-only' } @handle_rbd_mirror_error() diff --git a/src/pybind/rbd/c_rbd.pxd b/src/pybind/rbd/c_rbd.pxd index dec945969e23c..2df23aba6889e 100644 --- a/src/pybind/rbd/c_rbd.pxd +++ b/src/pybind/rbd/c_rbd.pxd @@ -136,6 +136,7 @@ cdef extern from "rbd/librbd.h" nogil: _RBD_MIRROR_MODE_DISABLED "RBD_MIRROR_MODE_DISABLED" _RBD_MIRROR_MODE_IMAGE "RBD_MIRROR_MODE_IMAGE" _RBD_MIRROR_MODE_POOL "RBD_MIRROR_MODE_POOL" + _RBD_MIRROR_MODE_INIT_ONLY "RBD_MIRROR_MODE_INIT_ONLY" ctypedef enum rbd_mirror_peer_direction_t: _RBD_MIRROR_PEER_DIRECTION_RX "RBD_MIRROR_PEER_DIRECTION_RX" diff --git a/src/pybind/rbd/mock_rbd.pxi b/src/pybind/rbd/mock_rbd.pxi index aa5e1609d82ce..b66de38a4aa23 100644 --- a/src/pybind/rbd/mock_rbd.pxi +++ b/src/pybind/rbd/mock_rbd.pxi @@ -140,6 +140,7 @@ cdef nogil: _RBD_MIRROR_MODE_DISABLED "RBD_MIRROR_MODE_DISABLED" _RBD_MIRROR_MODE_IMAGE "RBD_MIRROR_MODE_IMAGE" _RBD_MIRROR_MODE_POOL "RBD_MIRROR_MODE_POOL" + _RBD_MIRROR_MODE_INIT_ONLY "RBD_MIRROR_MODE_INIT_ONLY" ctypedef enum rbd_mirror_peer_direction_t: _RBD_MIRROR_PEER_DIRECTION_RX "RBD_MIRROR_PEER_DIRECTION_RX" diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index bcae8cc289cf1..e07b891dd20d4 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -86,6 +86,7 @@ RBD_FLAG_FAST_DIFF_INVALID = _RBD_FLAG_FAST_DIFF_INVALID RBD_MIRROR_MODE_DISABLED = _RBD_MIRROR_MODE_DISABLED RBD_MIRROR_MODE_IMAGE = _RBD_MIRROR_MODE_IMAGE RBD_MIRROR_MODE_POOL = _RBD_MIRROR_MODE_POOL +RBD_MIRROR_MODE_INIT_ONLY = _RBD_MIRROR_MODE_INIT_ONLY RBD_MIRROR_PEER_DIRECTION_RX = _RBD_MIRROR_PEER_DIRECTION_RX RBD_MIRROR_PEER_DIRECTION_TX = _RBD_MIRROR_PEER_DIRECTION_TX diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 5f30425835876..b94bd9bd7eeb1 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -1840,7 +1840,7 @@ Positional arguments pool specification (example: [/] - mirror mode [image or pool] + mirror mode [image, pool or init-only] Optional arguments -p [ --pool ] arg pool name diff --git a/src/test/cli/rbd/not-enough-args.t b/src/test/cli/rbd/not-enough-args.t index 3f613e6bbd8bb..749f095e7da46 100644 --- a/src/test/cli/rbd/not-enough-args.t +++ b/src/test/cli/rbd/not-enough-args.t @@ -197,7 +197,7 @@ rbd: image name was not specified [22] $ rbd mirror pool enable rbd - rbd: must specify 'image' or 'pool' mode. + rbd: mirror mode was not specified [22] $ rbd mirror pool peer add rbd rbd: remote cluster was not specified diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index 15673672fc560..7f8eb0440e0c2 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -29,8 +29,9 @@ from rbd import (RBD, Group, Image, ImageNotFound, InvalidArgument, ImageExists, RBD_FEATURE_DEEP_FLATTEN, RBD_FEATURE_FAST_DIFF, RBD_FEATURE_OBJECT_MAP, RBD_MIRROR_MODE_DISABLED, RBD_MIRROR_MODE_IMAGE, - RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_ENABLED, - RBD_MIRROR_IMAGE_DISABLED, MIRROR_IMAGE_STATUS_STATE_UNKNOWN, + RBD_MIRROR_MODE_POOL, RBD_MIRROR_MODE_INIT_ONLY, + RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_DISABLED, + MIRROR_IMAGE_STATUS_STATE_UNKNOWN, RBD_MIRROR_IMAGE_MODE_JOURNAL, RBD_MIRROR_IMAGE_MODE_SNAPSHOT, RBD_LOCK_MODE_EXCLUSIVE, RBD_OPERATION_FEATURE_GROUP, RBD_OPERATION_FEATURE_CLONE_CHILD, @@ -2393,23 +2394,90 @@ class TestMirroring(object): eq(rados.get_fsid(), self.rbd.mirror_site_name_get(rados)) def test_mirror_remote_namespace(self): - remote_namespace = "remote-ns" - # cannot set remote namespace for the default namespace + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # cannot set remote namespace while mirroring enabled + assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, + ioctx, "remote-ns1") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, - ioctx, remote_namespace) + ioctx, "") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_INIT_ONLY) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # set remote namespace for the default namespace while in init-only + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns1") + eq("remote-ns1", self.rbd.mirror_remote_namespace_get(ioctx)) + # reset remote namespace for the default namespace while in init-only + self.rbd.mirror_remote_namespace_set(ioctx, "") eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # change remote namespace for the default namespace while in init-only + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns2") + eq("remote-ns2", self.rbd.mirror_remote_namespace_get(ioctx)) + # disabling mirroring also resets remote namespace + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # set remote namespace for the default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns3") + eq("remote-ns3", self.rbd.mirror_remote_namespace_get(ioctx)) + # reset remote namespace for the default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # change remote namespace for the default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns4") + eq("remote-ns4", self.rbd.mirror_remote_namespace_get(ioctx)) + # moving to init-only or an enabled state preserves remote namespace + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_INIT_ONLY) + eq("remote-ns4", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL) + eq("remote-ns4", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_IMAGE) + eq("remote-ns4", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.namespace_create(ioctx, "ns1") ioctx.set_namespace("ns1") + eq("ns1", self.rbd.mirror_remote_namespace_get(ioctx)) + # set remote namespace on a non-default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns1") + eq("remote-ns1", self.rbd.mirror_remote_namespace_get(ioctx)) + # reset remote namespace on a non-default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "ns1") + eq("ns1", self.rbd.mirror_remote_namespace_get(ioctx)) + # change remote namespace on a non-default namespace while mirroring disabled + self.rbd.mirror_remote_namespace_set(ioctx, "remote-ns2") + eq("remote-ns2", self.rbd.mirror_remote_namespace_get(ioctx)) + # cannot move to init-only on a non-default namespace + assert_raises(InvalidArgument, self.rbd.mirror_mode_set, + ioctx, RBD_MIRROR_MODE_INIT_ONLY) + eq(RBD_MIRROR_MODE_DISABLED, self.rbd.mirror_mode_get(ioctx)) + # moving to an enabled state preserves remote namespace self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_IMAGE) + eq("remote-ns2", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL) + eq("remote-ns2", self.rbd.mirror_remote_namespace_get(ioctx)) + # disabling mirroring also resets remote namespace + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED) + eq("ns1", self.rbd.mirror_remote_namespace_get(ioctx)) + # set remote namespace on a non-default namespace to the default namespace + self.rbd.mirror_remote_namespace_set(ioctx, "") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # moving to an enabled state preserves remote namespace + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_IMAGE) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) # cannot set remote namespace while mirroring enabled assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, - ioctx, remote_namespace) - self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED) - # cannot set remote namespace to the default namespace + ioctx, "remote-ns1") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, + ioctx, "ns1") + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, ioctx, "") - self.rbd.mirror_remote_namespace_set(ioctx, remote_namespace) - eq(remote_namespace, self.rbd.mirror_remote_namespace_get(ioctx)) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + # disabling mirroring clears remote namespace + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED) + eq("ns1", self.rbd.mirror_remote_namespace_get(ioctx)) ioctx.set_namespace("") self.rbd.namespace_remove(ioctx, "ns1") diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 31540021eb5a4..58d32efdb2562 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -20,6 +20,8 @@ #include "tools/rbd_mirror/Threads.h" #include "common/Formatter.h" +#include + namespace librbd { namespace { @@ -347,28 +349,22 @@ public: Return(r))); } - void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl, - cls::rbd::MirrorMode mirror_mode, int r) { - bufferlist out_bl; - encode(mirror_mode, out_bl); - - EXPECT_CALL(*io_ctx_impl, - exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"), - _, _, _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { - *bl = out_bl; - })), - Return(r))); - } - - void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl) { + void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl* io_ctx_impl, + std::atomic* default_namespace_enabled, + int r) { EXPECT_CALL(*io_ctx_impl, exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"), _, _, _, _)) - .WillRepeatedly(DoAll(WithArg<5>(Invoke([](bufferlist *bl) { - encode(cls::rbd::MIRROR_MODE_POOL, *bl); + .WillRepeatedly(DoAll(WithArg<5>(Invoke( + [io_ctx_impl, default_namespace_enabled](bufferlist* bl) { + if (io_ctx_impl->get_namespace().empty() && + *default_namespace_enabled == false) { + encode(cls::rbd::MIRROR_MODE_DISABLED, *bl); + } else { + encode(cls::rbd::MIRROR_MODE_POOL, *bl); + } })), - Return(0))); + Return(r))); } void expect_mirror_remote_namespace_get( @@ -592,10 +588,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { peer_spec.mon_host = "123"; peer_spec.key = "234"; - auto mock_default_namespace_replayer = new MockNamespaceReplayer(); - expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, - false); - MockThreads mock_threads(m_threads); expect_work_queue(mock_threads); @@ -624,7 +616,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { auto mock_remote_pool_poller = new MockRemotePoolPoller(); expect_remote_pool_poller_init(*mock_remote_pool_poller, {"remote mirror uuid", ""}, 0); - expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; @@ -643,7 +634,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { remote_cct->put(); expect_leader_watcher_shut_down(*mock_leader_watcher); - expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); @@ -651,12 +641,6 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; - peer_spec.mon_host = "123"; - peer_spec.key = "234"; - - auto mock_default_namespace_replayer = new MockNamespaceReplayer(); - expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, - false); MockThreads mock_threads(m_threads); expect_work_queue(mock_threads); @@ -686,7 +670,6 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { auto mock_remote_pool_poller = new MockRemotePoolPoller(); expect_remote_pool_poller_init(*mock_remote_pool_poller, {"remote mirror uuid", ""}, 0); - expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; @@ -699,6 +682,86 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + + C_SaferCond on_acquire; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + expect_service_daemon_remove_attribute(mock_service_daemon, + SERVICE_DAEMON_LEADER_KEY); + + C_SaferCond on_release; + mock_leader_watcher->listener->pre_release_handler(&on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +TEST_F(TestMockPoolReplayer, DefaultNamespaceEnabled) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + + std::atomic default_namespace_enabled = true; + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + expect_namespace_replayer_get_remote_namespace(*mock_default_namespace_replayer, + ""); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + + expect_clone(mock_local_io_ctx); + expect_mirror_mode_get(mock_local_io_ctx, &default_namespace_enabled, 0); + expect_mirror_remote_namespace_get(mock_local_io_ctx, "", -ENOENT); + + InSequence seq; + + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + nullptr); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + C_SaferCond on_default_namespace_init; + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, ""); + expect_namespace_replayer_handle_update_leader(*mock_default_namespace_replayer, + "", &on_default_namespace_init); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + ASSERT_EQ(0, on_default_namespace_init.wait()); + expect_service_daemon_add_or_update_attribute( mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); expect_namespace_replayer_handle_acquire_leader( @@ -717,8 +780,103 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { mock_leader_watcher->listener->pre_release_handler(&on_release); ASSERT_EQ(0, on_release.wait()); - expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_service_daemon_remove_namespace(mock_service_daemon, ""); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +TEST_F(TestMockPoolReplayer, DefaultNamespaceEnableDisable) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + + g_ceph_context->_conf.set_val( + "rbd_mirror_pool_replayers_refresh_interval", "1"); + + std::atomic default_namespace_enabled = false; + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + expect_namespace_replayer_get_remote_namespace(*mock_default_namespace_replayer, + ""); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + + expect_clone(mock_local_io_ctx); + expect_mirror_mode_get(mock_local_io_ctx, &default_namespace_enabled, 0); + expect_mirror_remote_namespace_get(mock_local_io_ctx, "", -ENOENT); + + InSequence seq; + + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + nullptr); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + + C_SaferCond on_acquire; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + C_SaferCond on_default_namespace_acquire; + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, ""); + expect_namespace_replayer_handle_acquire_leader( + *mock_default_namespace_replayer, 0, &on_default_namespace_acquire); + expect_namespace_replayer_handle_instances_added( + *mock_default_namespace_replayer); + + default_namespace_enabled = true; + ASSERT_EQ(0, on_default_namespace_acquire.wait()); + + C_SaferCond on_default_namespace_shut_down; + expect_service_daemon_remove_namespace(mock_service_daemon, ""); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer, + &on_default_namespace_shut_down); + + default_namespace_enabled = false; + ASSERT_EQ(0, on_default_namespace_shut_down.wait()); + + expect_service_daemon_remove_attribute(mock_service_daemon, + SERVICE_DAEMON_LEADER_KEY); + + C_SaferCond on_release; + mock_leader_watcher->listener->pre_release_handler(&on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_leader_watcher_shut_down(*mock_leader_watcher); expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); @@ -726,17 +884,18 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { TEST_F(TestMockPoolReplayer, Namespaces) { PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; - peer_spec.mon_host = "123"; - peer_spec.key = "234"; g_ceph_context->_conf.set_val( "rbd_mirror_pool_replayers_refresh_interval", "1"); MockNamespace mock_namespace; + std::atomic default_namespace_enabled = true; auto mock_default_namespace_replayer = new MockNamespaceReplayer(); expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, false); + expect_namespace_replayer_get_remote_namespace(*mock_default_namespace_replayer, + ""); auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1"); expect_namespace_replayer_is_blocklisted(*mock_ns1_namespace_replayer, @@ -767,7 +926,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { g_ceph_context); expect_clone(mock_local_io_ctx); - expect_mirror_mode_get(mock_local_io_ctx); + expect_mirror_mode_get(mock_local_io_ctx, &default_namespace_enabled, 0); expect_mirror_remote_namespace_get(mock_local_io_ctx, "", -ENOENT); InSequence seq; @@ -780,7 +939,6 @@ TEST_F(TestMockPoolReplayer, Namespaces) { auto mock_remote_pool_poller = new MockRemotePoolPoller(); expect_remote_pool_poller_init(*mock_remote_pool_poller, {"remote mirror uuid", ""}, 0); - expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; @@ -788,11 +946,19 @@ TEST_F(TestMockPoolReplayer, Namespaces) { expect_service_daemon_add_or_update_instance_id_attribute( mock_service_daemon, instance_id); + C_SaferCond on_default_namespace_init; + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, ""); + expect_namespace_replayer_handle_update_leader(*mock_default_namespace_replayer, + "", &on_default_namespace_init); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); + ASSERT_EQ(0, on_default_namespace_init.wait()); + C_SaferCond on_ns1_init; expect_namespace_replayer_init(*mock_ns1_namespace_replayer, 0); expect_service_daemon_add_namespace(mock_service_daemon, "ns1"); @@ -813,9 +979,9 @@ TEST_F(TestMockPoolReplayer, Namespaces) { mock_leader_watcher->listener->post_acquire_handler(&on_acquire); ASSERT_EQ(0, on_acquire.wait()); + C_SaferCond on_ns2_acquire; expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); - C_SaferCond on_ns2_acquire; expect_namespace_replayer_handle_acquire_leader( *mock_ns2_namespace_replayer, 0, &on_ns2_acquire); expect_namespace_replayer_handle_instances_added( @@ -828,6 +994,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, &on_ns2_shut_down); + mock_namespace.remove("ns2"); ASSERT_EQ(0, on_ns2_shut_down.wait()); @@ -842,10 +1009,11 @@ TEST_F(TestMockPoolReplayer, Namespaces) { mock_leader_watcher->listener->pre_release_handler(&on_release); ASSERT_EQ(0, on_release.wait()); + expect_service_daemon_remove_namespace(mock_service_daemon, ""); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); expect_service_daemon_remove_namespace(mock_service_daemon, "ns1"); expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer); expect_leader_watcher_shut_down(*mock_leader_watcher); - expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); @@ -853,21 +1021,27 @@ TEST_F(TestMockPoolReplayer, Namespaces) { TEST_F(TestMockPoolReplayer, NamespacesError) { PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; - peer_spec.mon_host = "123"; - peer_spec.key = "234"; g_ceph_context->_conf.set_val( "rbd_mirror_pool_replayers_refresh_interval", "1"); MockNamespace mock_namespace; + std::atomic default_namespace_enabled = true; auto mock_default_namespace_replayer = new MockNamespaceReplayer(); expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, false); + expect_namespace_replayer_get_remote_namespace(*mock_default_namespace_replayer, + ""); + auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1"); + auto mock_ns2_namespace_replayer = new MockNamespaceReplayer("ns2"); expect_namespace_replayer_is_blocklisted(*mock_ns2_namespace_replayer, false); + expect_namespace_replayer_get_remote_namespace(*mock_ns2_namespace_replayer, + "ns2"); + auto mock_ns3_namespace_replayer = new MockNamespaceReplayer("ns3"); MockThreads mock_threads(m_threads); @@ -887,7 +1061,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { g_ceph_context); expect_clone(mock_local_io_ctx); - expect_mirror_mode_get(mock_local_io_ctx); + expect_mirror_mode_get(mock_local_io_ctx, &default_namespace_enabled, 0); expect_mirror_remote_namespace_get(mock_local_io_ctx, "", -ENOENT); InSequence seq; @@ -900,7 +1074,6 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { auto mock_remote_pool_poller = new MockRemotePoolPoller(); expect_remote_pool_poller_init(*mock_remote_pool_poller, {"remote mirror uuid", ""}, 0); - expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); MockServiceDaemon mock_service_daemon; @@ -908,11 +1081,19 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { expect_service_daemon_add_or_update_instance_id_attribute( mock_service_daemon, instance_id); + C_SaferCond on_default_namespace_init; + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, ""); + expect_namespace_replayer_handle_update_leader(*mock_default_namespace_replayer, + "", &on_default_namespace_init); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); + ASSERT_EQ(0, on_default_namespace_init.wait()); + // test namespace replayer init fails for non leader C_SaferCond on_ns1_init; @@ -922,48 +1103,40 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { on_ns1_init.complete(r); }); expect_namespace_replayer_init(*mock_ns1_namespace_replayer, -EINVAL, ctx); + mock_namespace.add("ns1"); ASSERT_EQ(-EINVAL, on_ns1_init.wait()); - // test acquire leader fails when default namespace replayer fails - - expect_service_daemon_add_or_update_attribute( - mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); - expect_namespace_replayer_handle_acquire_leader( - *mock_default_namespace_replayer, -EINVAL); - - C_SaferCond on_acquire1; - mock_leader_watcher->listener->post_acquire_handler(&on_acquire1); - ASSERT_EQ(-EINVAL, on_acquire1.wait()); - - // test acquire leader succeeds when non-default namespace replayer fails + // test acquire leader succeeds even when all namespace replayers fail C_SaferCond on_ns2_init; expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_handle_update_leader(*mock_ns2_namespace_replayer, "", &on_ns2_init); + mock_namespace.add("ns2"); ASSERT_EQ(0, on_ns2_init.wait()); expect_service_daemon_add_or_update_attribute( mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); expect_namespace_replayer_handle_acquire_leader( - *mock_default_namespace_replayer, 0); - + *mock_default_namespace_replayer, -EPERM); expect_namespace_replayer_handle_acquire_leader(*mock_ns2_namespace_replayer, -EINVAL); ctx = new LambdaContext( - [&mock_namespace](int) { + [&mock_namespace, &default_namespace_enabled](int) { + default_namespace_enabled = false; mock_namespace.remove("ns2"); }); + expect_service_daemon_remove_namespace(mock_service_daemon, ""); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer, nullptr); expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, ctx); - mock_namespace.add("ns2"); - C_SaferCond on_acquire2; - mock_leader_watcher->listener->post_acquire_handler(&on_acquire2); - ASSERT_EQ(0, on_acquire2.wait()); + C_SaferCond on_acquire; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); // test namespace replayer init fails on acquire leader @@ -979,11 +1152,11 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { -EINVAL); expect_service_daemon_remove_namespace(mock_service_daemon, "ns3"); expect_namespace_replayer_shut_down(*mock_ns3_namespace_replayer, ctx); + mock_namespace.add("ns3"); ASSERT_EQ(0, on_ns3_shut_down.wait()); expect_leader_watcher_shut_down(*mock_leader_watcher); - expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); @@ -992,10 +1165,6 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { TEST_F(TestMockPoolReplayer, RemoveCalloutOnInit) { PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; - auto mock_default_namespace_replayer = new MockNamespaceReplayer(); - expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, - false); - auto mock_leader_watcher = new MockLeaderWatcher(); expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); @@ -1051,7 +1220,6 @@ TEST_F(TestMockPoolReplayer, RemoveCalloutOnInit) { expect_remote_pool_poller_init(*mock_remote_pool_poller, {"remote mirror uuid", ""}, 0); - expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); expect_service_daemon_remove_callout(mock_service_daemon, 123); @@ -1062,7 +1230,6 @@ TEST_F(TestMockPoolReplayer, RemoveCalloutOnInit) { pool_replayer.init("siteA"); expect_leader_watcher_shut_down(*mock_leader_watcher); - expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc index a7f16dba1b40c..0a0b43afe3db7 100644 --- a/src/tools/rbd/action/MirrorPool.cc +++ b/src/tools/rbd/action/MirrorPool.cc @@ -1255,7 +1255,7 @@ void get_enable_arguments(po::options_description *positional, po::options_description *options) { at::add_pool_options(positional, options, true); positional->add_options() - ("mode", "mirror mode [image or pool]"); + ("mode", "mirror mode [image, pool or init-only]"); add_site_name_optional(options); options->add_options() @@ -1343,8 +1343,10 @@ int execute_enable(const po::variables_map &vm, mirror_mode = RBD_MIRROR_MODE_IMAGE; } else if (mode == "pool") { mirror_mode = RBD_MIRROR_MODE_POOL; + } else if (mode == "init-only") { + mirror_mode = RBD_MIRROR_MODE_INIT_ONLY; } else { - std::cerr << "rbd: must specify 'image' or 'pool' mode." << std::endl; + std::cerr << "rbd: mirror mode was not specified" << std::endl; return -EINVAL; } @@ -1357,27 +1359,35 @@ int execute_enable(const po::variables_map &vm, } if (vm.count(REMOTE_NAMESPACE_NAME)) { + if (mirror_mode == RBD_MIRROR_MODE_INIT_ONLY) { + std::cerr << "rbd: cannot specify remote namespace for init-only mode" + << std::endl; + return -EINVAL; + } remote_namespace = vm[REMOTE_NAMESPACE_NAME].as(); } else { remote_namespace = namespace_name; } - std::string original_remote_namespace; librbd::RBD rbd; - r = rbd.mirror_remote_namespace_get(io_ctx, &original_remote_namespace); - if (r < 0) { - std::cerr << "rbd: failed to get the current remote namespace." - << std::endl; - return r; - } - if (original_remote_namespace != remote_namespace) { - r = rbd.mirror_remote_namespace_set(io_ctx, remote_namespace); + if (mirror_mode != RBD_MIRROR_MODE_INIT_ONLY) { + std::string original_remote_namespace; + r = rbd.mirror_remote_namespace_get(io_ctx, &original_remote_namespace); if (r < 0) { - std::cerr << "rbd: failed to set the remote namespace." - << std::endl; + std::cerr << "rbd: failed to get the current remote namespace." + << std::endl; return r; } + + if (original_remote_namespace != remote_namespace) { + r = rbd.mirror_remote_namespace_set(io_ctx, remote_namespace); + if (r < 0) { + std::cerr << "rbd: failed to set the remote namespace." + << std::endl; + return r; + } + } } bool updated = false; @@ -1472,6 +1482,9 @@ int execute_info(const po::variables_map &vm, case RBD_MIRROR_MODE_POOL: mirror_mode_desc = "pool"; break; + case RBD_MIRROR_MODE_INIT_ONLY: + mirror_mode_desc = "init-only"; + break; default: mirror_mode_desc = "unknown"; break; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 3f558e8ddba58..e0c24535add0e 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -130,6 +130,10 @@ void NamespaceReplayer::print_status(Formatter *f) std::lock_guard locker{m_lock}; + f->open_object_section("namespace_replayer_status"); + f->dump_string("local_namespace", m_local_namespace_name); + f->dump_string("remote_namespace", m_remote_namespace_name); + m_instance_replayer->print_status(f); if (m_image_deleter) { @@ -137,6 +141,8 @@ void NamespaceReplayer::print_status(Formatter *f) m_image_deleter->print_status(f); f->close_section(); } + + f->close_section(); // namespace_replayer_status } template diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 2efb8bae38b4e..19bc52fd86475 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -368,25 +368,6 @@ void PoolReplayer::init(const std::string& site_name) { m_pool_meta_cache->set_local_pool_meta( m_local_io_ctx.get_id(), {m_local_mirror_uuid}); - m_default_namespace_replayer.reset(NamespaceReplayer::create( - "", "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, - m_remote_pool_meta, m_threads, m_image_sync_throttler.get(), - m_image_deletion_throttler.get(), m_service_daemon, - m_cache_manager_handler, m_pool_meta_cache)); - - C_SaferCond on_init; - m_default_namespace_replayer->init(&on_init); - r = on_init.wait(); - if (r < 0) { - derr << "error initializing default namespace replayer: " << cpp_strerror(r) - << dendl; - m_callout_id = m_service_daemon->add_or_update_callout( - m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR, - "unable to initialize default namespace replayer"); - m_default_namespace_replayer.reset(); - return; - } - m_leader_watcher.reset(LeaderWatcher::create(m_threads, m_local_io_ctx, &m_leader_listener)); r = m_leader_watcher->init(); @@ -428,13 +409,6 @@ void PoolReplayer::shut_down() { } m_leader_watcher.reset(); - if (m_default_namespace_replayer) { - C_SaferCond on_shut_down; - m_default_namespace_replayer->shut_down(&on_shut_down); - on_shut_down.wait(); - } - m_default_namespace_replayer.reset(); - if (m_remote_pool_poller) { C_SaferCond ctx; m_remote_pool_poller->shut_down(&ctx); @@ -602,10 +576,10 @@ void PoolReplayer::run() { std::unique_lock locker{m_lock}; - if (m_leader_watcher->is_blocklisted() || - m_default_namespace_replayer->is_blocklisted()) { + if (m_leader_watcher->is_blocklisted()) { m_blocklisted = true; m_stopping = true; + break; } for (auto &it : m_namespace_replayers) { @@ -680,8 +654,8 @@ void PoolReplayer::update_namespace_replayers() { ctx=gather_ctx->new_sub()](int r) { std::lock_guard locker{m_lock}; if (r < 0) { - derr << "failed to initialize namespace replayer for namespace " - << names.first << ": " << cpp_strerror(r) << dendl; + derr << "failed to initialize namespace replayer for namespace '" + << names.first << "': " << cpp_strerror(r) << dendl; delete namespace_replayer; mirroring_namespaces.erase(names.first); } else { @@ -740,43 +714,47 @@ int PoolReplayer::list_mirroring_namespaces( ceph_assert(ceph_mutex_is_locked(m_lock)); std::vector names; - int r = librbd::api::Namespace::list(m_local_io_ctx, &names); if (r < 0) { derr << "failed to list namespaces: " << cpp_strerror(r) << dendl; return r; } + // handle the default namespace the same way + names.push_back(""); + for (auto &name : names) { librados::IoCtx ns_ioctx; ns_ioctx.dup(m_local_io_ctx); ns_ioctx.set_namespace(name); - std::string remote_namespace; cls::rbd::MirrorMode mirror_mode = cls::rbd::MIRROR_MODE_DISABLED; int r = librbd::cls_client::mirror_mode_get(&ns_ioctx, &mirror_mode); if (r < 0 && r != -ENOENT) { - derr << "failed to get namespace mirror mode: " << cpp_strerror(r) - << dendl; + derr << "failed to get mirror mode for namespace '" << name << "': " + << cpp_strerror(r) << dendl; if (m_namespace_replayers.count(name) == 0) { continue; } - } else if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { + } else if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED || + mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY) { dout(10) << "mirroring is disabled for namespace " << name << dendl; continue; } + + std::string remote_namespace; r = librbd::cls_client::mirror_remote_namespace_get(&ns_ioctx, &remote_namespace); - if (r < 0) { + if (r < 0) { if (r != -ENOENT && r != -EOPNOTSUPP) { - derr << "failed to get remote mirror namespace: " << cpp_strerror(r) - << dendl; + derr << "failed to get remote namespace for namespace '" << name + << "': " << cpp_strerror(r) << dendl; continue; } else { remote_namespace = name; } } - + dout(10) << " local namespace=" << name << ", remote namespace=" << remote_namespace << dendl; namespaces->insert(std::make_pair(name, remote_namespace)); @@ -811,8 +789,8 @@ void PoolReplayer::namespace_replayer_acquire_leader(const std::string &name, on_finish = new LambdaContext( [this, name, on_finish](int r) { if (r < 0) { - derr << "failed to handle acquire leader for namespace: " - << name << ": " << cpp_strerror(r) << dendl; + derr << "failed to handle acquire leader for namespace '" + << name << "': " << cpp_strerror(r) << dendl; // remove the namespace replayer -- update_namespace_replayers will // retry to create it and acquire leader. @@ -902,18 +880,11 @@ void PoolReplayer::print_status(Formatter *f) { f->close_section(); // deletion_throttler } - if (m_default_namespace_replayer) { - m_default_namespace_replayer->print_status(f); - } - - f->open_array_section("namespaces"); + f->open_array_section("namespace_replayers"); for (auto &it : m_namespace_replayers) { - f->open_object_section("namespace"); - f->dump_string("name", it.first); it.second->print_status(f); - f->close_section(); // namespace } - f->close_section(); // namespaces + f->close_section(); // namespace_replayers f->close_section(); // pool_replayer_status } @@ -930,9 +901,6 @@ void PoolReplayer::start() { m_manual_stop = false; - if (m_default_namespace_replayer) { - m_default_namespace_replayer->start(); - } for (auto &it : m_namespace_replayers) { it.second->start(); } @@ -953,9 +921,6 @@ void PoolReplayer::stop(bool manual) { m_manual_stop = true; - if (m_default_namespace_replayer) { - m_default_namespace_replayer->stop(); - } for (auto &it : m_namespace_replayers) { it.second->stop(); } @@ -971,9 +936,6 @@ void PoolReplayer::restart() { return; } - if (m_default_namespace_replayer) { - m_default_namespace_replayer->restart(); - } for (auto &it : m_namespace_replayers) { it.second->restart(); } @@ -989,9 +951,6 @@ void PoolReplayer::flush() { return; } - if (m_default_namespace_replayer) { - m_default_namespace_replayer->flush(); - } for (auto &it : m_namespace_replayers) { it.second->flush(); } @@ -1023,21 +982,19 @@ void PoolReplayer::handle_post_acquire_leader(Context *on_finish) { m_service_daemon->add_or_update_attribute(m_local_pool_id, SERVICE_DAEMON_LEADER_KEY, true); - auto ctx = new LambdaContext( + auto ctx = librbd::util::create_async_context_callback( + m_threads->work_queue, new LambdaContext( [this, on_finish](int r) { if (r == 0) { std::lock_guard locker{m_lock}; m_leader = true; } on_finish->complete(r); - }); + })); auto cct = reinterpret_cast(m_local_io_ctx.cct()); auto gather_ctx = new C_Gather(cct, ctx); - m_default_namespace_replayer->handle_acquire_leader( - gather_ctx->new_sub()); - for (auto &it : m_namespace_replayers) { namespace_replayer_acquire_leader(it.first, gather_ctx->new_sub()); } @@ -1059,12 +1016,11 @@ void PoolReplayer::handle_pre_release_leader(Context *on_finish) { m_leader = false; m_service_daemon->remove_attribute(m_local_pool_id, SERVICE_DAEMON_LEADER_KEY); + auto ctx = librbd::util::create_async_context_callback( + m_threads->work_queue, on_finish); auto cct = reinterpret_cast(m_local_io_ctx.cct()); - auto gather_ctx = new C_Gather(cct, on_finish); - - m_default_namespace_replayer->handle_release_leader( - gather_ctx->new_sub()); + auto gather_ctx = new C_Gather(cct, ctx); for (auto &it : m_namespace_replayers) { it.second->handle_release_leader(gather_ctx->new_sub()); @@ -1081,8 +1037,6 @@ void PoolReplayer::handle_update_leader( std::lock_guard locker{m_lock}; - m_default_namespace_replayer->handle_update_leader(leader_instance_id); - for (auto &it : m_namespace_replayers) { it.second->handle_update_leader(leader_instance_id); } @@ -1098,8 +1052,6 @@ void PoolReplayer::handle_instances_added( return; } - m_default_namespace_replayer->handle_instances_added(instance_ids); - for (auto &it : m_namespace_replayers) { it.second->handle_instances_added(instance_ids); } @@ -1115,8 +1067,6 @@ void PoolReplayer::handle_instances_removed( return; } - m_default_namespace_replayer->handle_instances_removed(instance_ids); - for (auto &it : m_namespace_replayers) { it.second->handle_instances_removed(instance_ids); } @@ -1127,7 +1077,7 @@ void PoolReplayer::handle_remote_pool_meta_updated( const RemotePoolMeta& remote_pool_meta) { dout(5) << "remote_pool_meta=" << remote_pool_meta << dendl; - if (!m_default_namespace_replayer) { + if (!m_leader_watcher) { m_remote_pool_meta = remote_pool_meta; return; } diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 4635f1ae6bac2..7e673e5d5e5a0 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -219,7 +219,6 @@ private: std::unique_ptr m_remote_pool_poller_listener; std::unique_ptr> m_remote_pool_poller; - std::unique_ptr> m_default_namespace_replayer; std::map *> m_namespace_replayers; std::string m_asok_hook_name; diff --git a/src/tools/rbd_mirror/ServiceDaemon.cc b/src/tools/rbd_mirror/ServiceDaemon.cc index 9ef716fd8e25c..2276c2ec57199 100644 --- a/src/tools/rbd_mirror/ServiceDaemon.cc +++ b/src/tools/rbd_mirror/ServiceDaemon.cc @@ -208,11 +208,6 @@ template void ServiceDaemon::add_or_update_namespace_attribute( int64_t pool_id, const std::string& namespace_name, const std::string& key, const AttributeValue& value) { - if (namespace_name.empty()) { - add_or_update_attribute(pool_id, key, value); - return; - } - dout(20) << "pool_id=" << pool_id << ", " << "namespace=" << namespace_name << ", " << "key=" << key << ", " @@ -295,18 +290,17 @@ void ServiceDaemon::update_status() { boost::apply_visitor(attribute_dump_visitor, attribute.second); } - if (!pool_pair.second.ns_attributes.empty()) { - f.open_object_section("namespaces"); - for (auto& [ns, attributes] : pool_pair.second.ns_attributes) { - f.open_object_section(ns.c_str()); - for (auto& [key, value] : attributes) { - AttributeDumpVisitor attribute_dump_visitor(&f, key); - boost::apply_visitor(attribute_dump_visitor, value); - } - f.close_section(); // namespace + f.open_object_section("namespaces"); + for (auto& [ns, attributes] : pool_pair.second.ns_attributes) { + f.open_object_section(ns.c_str()); + for (auto& [key, value] : attributes) { + AttributeDumpVisitor attribute_dump_visitor(&f, key); + boost::apply_visitor(attribute_dump_visitor, value); } - f.close_section(); // namespaces + f.close_section(); // namespace } + f.close_section(); // namespaces + f.close_section(); // pool } f.close_section(); // pools -- 2.39.5