From 9e28b07f15281d49e9862598718b62c1e1abe8d7 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Fri, 21 Feb 2020 16:48:56 -0500 Subject: [PATCH] librbd: image-meta wrapper for snapshot-based mirroring properties The only property currently stored within the image-meta is the resync request flag. Signed-off-by: Jason Dillaman --- src/cls/rbd/cls_rbd_client.cc | 39 +++- src/cls/rbd/cls_rbd_client.h | 4 + src/librbd/CMakeLists.txt | 1 + src/librbd/mirror/snapshot/ImageMeta.cc | 174 ++++++++++++++++++ src/librbd/mirror/snapshot/ImageMeta.h | 78 ++++++++ src/librbd/mirror/snapshot/Utils.cc | 4 + src/librbd/mirror/snapshot/Utils.h | 4 + src/test/librbd/CMakeLists.txt | 1 + .../mirror/snapshot/test_mock_ImageMeta.cc | 159 ++++++++++++++++ 9 files changed, 454 insertions(+), 10 deletions(-) create mode 100644 src/librbd/mirror/snapshot/ImageMeta.cc create mode 100644 src/librbd/mirror/snapshot/ImageMeta.h create mode 100644 src/test/librbd/mirror/snapshot/test_mock_ImageMeta.cc diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index 808569b1ed5..4ec2d667242 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -1522,23 +1522,42 @@ int metadata_list_finish(bufferlist::const_iterator *it, return 0; } +void metadata_get_start(librados::ObjectReadOperation* op, + const std::string &key) { + bufferlist bl; + encode(key, bl); + + op->exec("rbd", "metadata_get", bl); +} + +int metadata_get_finish(bufferlist::const_iterator *it, + std::string* value) { + try { + decode(*value, *it); + } catch (const buffer::error &err) { + return -EBADMSG; + } + return 0; +} + int metadata_get(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, string *s) { ceph_assert(s); - bufferlist in, out; - encode(key, in); - int r = ioctx->exec(oid, "rbd", "metadata_get", in, out); - if (r < 0) - return r; + librados::ObjectReadOperation op; + metadata_get_start(&op, key); - auto iter = out.cbegin(); - try { - decode(*s, iter); - } catch (const buffer::error &err) { - return -EBADMSG; + bufferlist out_bl; + int r = ioctx->operate(oid, &op, &out_bl); + if (r < 0) { + return r; } + auto it = out_bl.cbegin(); + r = metadata_get_finish(&it, s); + if (r < 0) { + return r; + } return 0; } diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index ba9d94bb136..55353e0746f 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -249,6 +249,10 @@ void metadata_remove(librados::ObjectWriteOperation *op, const std::string &key); int metadata_remove(librados::IoCtx *ioctx, const std::string &oid, const std::string &key); +void metadata_get_start(librados::ObjectReadOperation* op, + const std::string &key); +int metadata_get_finish(bufferlist::const_iterator *it, + std::string* value); int metadata_get(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, string *v); diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index c2772dc580c..9eba807f75b 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -112,6 +112,7 @@ set(librbd_internal_srcs mirror/snapshot/CreatePrimaryRequest.cc mirror/snapshot/DemoteRequest.cc mirror/snapshot/GetImageStateRequest.cc + mirror/snapshot/ImageMeta.cc mirror/snapshot/PromoteRequest.cc mirror/snapshot/RemoveImageStateRequest.cc mirror/snapshot/SetImageStateRequest.cc diff --git a/src/librbd/mirror/snapshot/ImageMeta.cc b/src/librbd/mirror/snapshot/ImageMeta.cc new file mode 100644 index 00000000000..12607de2683 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.cc @@ -0,0 +1,174 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/WatchNotifyTypes.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/watcher/Notifier.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::ImageMeta: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; +using librbd::mirror::snapshot::util::get_image_meta_key; + +template +ImageMeta::ImageMeta(I* image_ctx, const std::string& mirror_uuid) + : m_image_ctx(image_ctx), m_mirror_uuid(mirror_uuid) { +} + +template +void ImageMeta::load(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_get_start(&op, get_image_meta_key(m_mirror_uuid)); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_load(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_load(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + std::string data; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = cls_client::metadata_get_finish(&it, &data); + } + + if (r == -ENOENT) { + ldout(m_image_ctx->cct, 15) << "no snapshot-based mirroring image-meta: " + << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } else if (r < 0) { + lderr(m_image_ctx->cct) << "failed to load snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + bool json_valid = false; + json_spirit::mValue json_root; + if (json_spirit::read(data, json_root)) { + try { + auto& json_obj = json_root.get_obj(); + resync_requested = json_obj["resync_requested"].get_bool(); + json_valid = true; + } catch (std::runtime_error&) { + } + } + + if (!json_valid) { + lderr(m_image_ctx->cct) << "invalid image-meta JSON received" << dendl; + on_finish->complete(-EBADMSG); + return; + } + + on_finish->complete(0); +} + +template +void ImageMeta::save(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + // simple implementation for now + std::string json = "{\"resync_requested\": " + + std::string(resync_requested ? "true" : "false") + "}"; + + bufferlist bl; + bl.append(json); + + // avoid using built-in metadata_set operation since that would require + // opening the non-primary image in read/write mode which isn't supported + librados::ObjectWriteOperation op; + cls_client::metadata_set(&op, {{get_image_meta_key(m_mirror_uuid), bl}}); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_save(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_save(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to save snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + notify_update(on_finish); +} + +template +void ImageMeta::notify_update(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << dendl; + + // directly send header notification on image since you cannot + // open a non-primary image read/write and therefore cannot re-use + // the ImageWatcher to send the notification + bufferlist bl; + encode(watch_notify::NotifyMessage(watch_notify::HeaderUpdatePayload()), bl); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_notify_update(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_notify( + m_image_ctx->header_oid, aio_comp, bl, watcher::Notifier::NOTIFY_TIMEOUT, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_notify_update(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to notify image update: " + << cpp_strerror(r) << dendl; + } + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::ImageMeta; diff --git a/src/librbd/mirror/snapshot/ImageMeta.h b/src/librbd/mirror/snapshot/ImageMeta.h new file mode 100644 index 00000000000..5d05f192797 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.h @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H + +#include "include/rados/librados.hpp" +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class ImageMeta { +public: + static ImageMeta* create(ImageCtxT* image_ctx, + const std::string& mirror_uuid) { + return new ImageMeta(image_ctx, mirror_uuid); + } + + ImageMeta(ImageCtxT* image_ctx, const std::string& mirror_uuid); + + void load(Context* on_finish); + void save(Context* on_finish); + + bool resync_requested = false; + +private: + /** + * @verbatim + * + * + * | + * v + * METADATA_GET + * | + * v + * + * | + * v + * METADATA_SET + * | + * v + * NOTIFY_UPDATE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT* m_image_ctx; + std::string m_mirror_uuid; + + bufferlist m_out_bl; + + void handle_load(Context* on_finish, int r); + + void handle_save(Context* on_finish, int r); + + void notify_update(Context* on_finish); + void handle_notify_update(Context* on_finish, int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::ImageMeta; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H diff --git a/src/librbd/mirror/snapshot/Utils.cc b/src/librbd/mirror/snapshot/Utils.cc index 6081be8c405..41629beab5d 100644 --- a/src/librbd/mirror/snapshot/Utils.cc +++ b/src/librbd/mirror/snapshot/Utils.cc @@ -48,6 +48,10 @@ bool get_rollback_snap_id( } // anonymous namespace +std::string get_image_meta_key(const std::string& mirror_uuid) { + return ".rbd_mirror." + mirror_uuid; +} + template bool can_create_primary_snapshot(I *image_ctx, bool demoted, bool force, uint64_t *rollback_snap_id) { diff --git a/src/librbd/mirror/snapshot/Utils.h b/src/librbd/mirror/snapshot/Utils.h index a6fce636ffb..86ecf04dfde 100644 --- a/src/librbd/mirror/snapshot/Utils.h +++ b/src/librbd/mirror/snapshot/Utils.h @@ -4,7 +4,9 @@ #ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H #define CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H +#include "include/int_types.h" #include "include/stringify.h" +#include namespace librbd { @@ -14,6 +16,8 @@ namespace mirror { namespace snapshot { namespace util { +std::string get_image_meta_key(const std::string& mirror_uuid); + template bool can_create_primary_snapshot(ImageCtxT *image_ctx, bool demoted, bool force, uint64_t *rollback_snap_id); diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index b0c68237e87..f1179f4f664 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -87,6 +87,7 @@ set(unittest_librbd_srcs managed_lock/test_mock_ReleaseRequest.cc mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc mirror/snapshot/test_mock_CreatePrimaryRequest.cc + mirror/snapshot/test_mock_ImageMeta.cc mirror/snapshot/test_mock_PromoteRequest.cc mirror/snapshot/test_mock_UnlinkPeerRequest.cc mirror/snapshot/test_mock_Utils.cc diff --git a/src/test/librbd/mirror/snapshot/test_mock_ImageMeta.cc b/src/test/librbd/mirror/snapshot/test_mock_ImageMeta.cc new file mode 100644 index 00000000000..9bc6139d83e --- /dev/null +++ b/src/test/librbd/mirror/snapshot/test_mock_ImageMeta.cc @@ -0,0 +1,159 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "test/librbd/test_support.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "librbd/ImageState.h" +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "librbd/mirror/snapshot/Utils.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx& image_ctx) : MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +#include "librbd/mirror/snapshot/ImageMeta.cc" + +namespace librbd { +namespace mirror { +namespace snapshot { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockMirrorSnapshotImageMeta : public TestMockFixture { +public: + typedef ImageMeta MockImageMeta; + + void expect_metadata_get(MockTestImageCtx& mock_image_ctx, + const std::string& mirror_uuid, + const std::string& value, int r) { + bufferlist in_bl; + ceph::encode(util::get_image_meta_key(mirror_uuid), in_bl); + + bufferlist out_bl; + ceph::encode(value, out_bl); + + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(mock_image_ctx.header_oid, _, StrEq("rbd"), + StrEq("metadata_get"), ContentsEqual(in_bl), _, _)) + .WillOnce(DoAll(WithArg<5>(CopyInBufferlist(out_bl)), + Return(r))); + } + + void expect_metadata_set(MockTestImageCtx& mock_image_ctx, + const std::string& mirror_uuid, + const std::string& value, int r) { + bufferlist value_bl; + value_bl.append(value); + + bufferlist in_bl; + ceph::encode( + std::map{ + {util::get_image_meta_key(mirror_uuid), value_bl}}, + in_bl); + + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(mock_image_ctx.header_oid, _, StrEq("rbd"), + StrEq("metadata_set"), ContentsEqual(in_bl), _, _)) + .WillOnce(Return(r)); + } +}; + +TEST_F(TestMockMirrorSnapshotImageMeta, Load) { + librbd::ImageCtx* image_ctx; + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + MockTestImageCtx mock_image_ctx(*image_ctx); + + InSequence seq; + expect_metadata_get(mock_image_ctx, "mirror uuid", + "{\"resync_requested\": true}", 0); + + MockImageMeta mock_image_meta(&mock_image_ctx, "mirror uuid"); + C_SaferCond ctx; + mock_image_meta.load(&ctx); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockMirrorSnapshotImageMeta, LoadError) { + librbd::ImageCtx* image_ctx; + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + MockTestImageCtx mock_image_ctx(*image_ctx); + + InSequence seq; + expect_metadata_get(mock_image_ctx, "mirror uuid", + "{\"resync_requested\": true}", -EINVAL); + + MockImageMeta mock_image_meta(&mock_image_ctx, "mirror uuid"); + C_SaferCond ctx; + mock_image_meta.load(&ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockMirrorSnapshotImageMeta, LoadCorrupt) { + librbd::ImageCtx* image_ctx; + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + MockTestImageCtx mock_image_ctx(*image_ctx); + + InSequence seq; + expect_metadata_get(mock_image_ctx, "mirror uuid", + "\"resync_requested\": true}", 0); + + MockImageMeta mock_image_meta(&mock_image_ctx, "mirror uuid"); + C_SaferCond ctx; + mock_image_meta.load(&ctx); + ASSERT_EQ(-EBADMSG, ctx.wait()); +} + +TEST_F(TestMockMirrorSnapshotImageMeta, Save) { + librbd::ImageCtx* image_ctx; + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + MockTestImageCtx mock_image_ctx(*image_ctx); + + InSequence seq; + expect_metadata_set(mock_image_ctx, "mirror uuid", + "{\"resync_requested\": true}", 0); + + MockImageMeta mock_image_meta(&mock_image_ctx, "mirror uuid"); + mock_image_meta.resync_requested = true; + + C_SaferCond ctx; + mock_image_meta.save(&ctx); + ASSERT_EQ(0, ctx.wait()); + + // should have sent image-update notification + ASSERT_TRUE(image_ctx->state->is_refresh_required()); +} + +TEST_F(TestMockMirrorSnapshotImageMeta, SaveError) { + librbd::ImageCtx* image_ctx; + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + MockTestImageCtx mock_image_ctx(*image_ctx); + + InSequence seq; + expect_metadata_set(mock_image_ctx, "mirror uuid", + "{\"resync_requested\": false}", -EINVAL); + + MockImageMeta mock_image_meta(&mock_image_ctx, "mirror uuid"); + + C_SaferCond ctx; + mock_image_meta.save(&ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd -- 2.39.5