From: Jason Dillaman Date: Wed, 27 Feb 2019 18:50:29 +0000 (-0500) Subject: librbd: separate pool validation into a standalone state machine X-Git-Tag: v12.2.13~216^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=49d3ec75b2c4bced867bd0427105928951974f19;p=ceph.git librbd: separate pool validation into a standalone state machine Signed-off-by: Jason Dillaman (cherry picked from commit b3ee83b9ce9698a933c784dbb4a9e19a81847a50) Conflicts: src/librbd/image/ValidatePoolRequest.cc: removed namespace support --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 072eaed491f..3ff52692f0f 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -40,6 +40,7 @@ set(librbd_internal_srcs image/RemoveRequest.cc image/SetFlagsRequest.cc image/SetSnapRequest.cc + image/ValidatePoolRequest.cc image_watcher/NotifyLockOwner.cc io/AioCompletion.cc io/AsyncOperation.cc diff --git a/src/librbd/image/ValidatePoolRequest.cc b/src/librbd/image/ValidatePoolRequest.cc new file mode 100644 index 00000000000..0b1a49e4216 --- /dev/null +++ b/src/librbd/image/ValidatePoolRequest.cc @@ -0,0 +1,233 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/image/ValidatePoolRequest.h" +#include "include/rados/librados.hpp" +#include "include/assert.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::image::ValidatePoolRequest: " \ + << __func__ << ": " + +namespace librbd { +namespace image { + +namespace { + +const std::string OVERWRITE_VALIDATED("overwrite validated"); +const std::string VALIDATE("validate"); + +} // anonymous namespace + +using util::create_rados_callback; +using util::create_context_callback; +using util::create_async_context_callback; + +template +ValidatePoolRequest::ValidatePoolRequest(librados::IoCtx& io_ctx, + ContextWQ *op_work_queue, + Context *on_finish) + : m_cct(reinterpret_cast(io_ctx.cct())), + m_op_work_queue(op_work_queue), m_on_finish(on_finish) { + m_io_ctx.dup(io_ctx); +} + +template +void ValidatePoolRequest::send() { + read_rbd_info(); +} + +template +void ValidatePoolRequest::read_rbd_info() { + ldout(m_cct, 5) << dendl; + + auto comp = create_rados_callback< + ValidatePoolRequest, + &ValidatePoolRequest::handle_read_rbd_info>(this); + + librados::ObjectReadOperation op; + op.read(0, 0, nullptr, nullptr); + + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(RBD_INFO, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void ValidatePoolRequest::handle_read_rbd_info(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + + if (r >= 0) { + bufferlist validated_bl; + validated_bl.append(OVERWRITE_VALIDATED); + + bufferlist validate_bl; + validate_bl.append(VALIDATE); + + if (m_out_bl.contents_equal(validated_bl)) { + // already validated pool + finish(0); + return; + } else if (m_out_bl.contents_equal(validate_bl)) { + // implies snapshot was already successfully created + overwrite_rbd_info(); + return; + } + } else if (r < 0 && r != -ENOENT) { + lderr(m_cct) << "failed to read RBD info: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_snapshot(); +} + +template +void ValidatePoolRequest::create_snapshot() { + ldout(m_cct, 5) << dendl; + + // allocate a self-managed snapshot id if this a new pool to force + // self-managed snapshot mode + auto ctx = new FunctionContext([this](int r) { + r = m_io_ctx.selfmanaged_snap_create(&m_snap_id); + handle_create_snapshot(r); + }); + m_op_work_queue->queue(ctx, 0); +} + +template +void ValidatePoolRequest::handle_create_snapshot(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + + if (r == -EINVAL) { + lderr(m_cct) << "pool not configured for self-managed RBD snapshot support" + << dendl; + finish(r); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to allocate self-managed snapshot: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + write_rbd_info(); +} + +template +void ValidatePoolRequest::write_rbd_info() { + ldout(m_cct, 5) << dendl; + + bufferlist bl; + bl.append(VALIDATE); + + librados::ObjectWriteOperation op; + op.create(true); + op.write(0, bl); + + auto comp = create_rados_callback< + ValidatePoolRequest, + &ValidatePoolRequest::handle_write_rbd_info>(this); + int r = m_io_ctx.aio_operate(RBD_INFO, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void ValidatePoolRequest::handle_write_rbd_info(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + + if (r == -EOPNOTSUPP) { + lderr(m_cct) << "pool missing required overwrite support" << dendl; + m_ret_val = -EINVAL; + } else if (r < 0 && r != -EEXIST) { + lderr(m_cct) << "failed to write RBD info: " << cpp_strerror(r) << dendl; + m_ret_val = r; + } + + remove_snapshot(); +} + +template +void ValidatePoolRequest::remove_snapshot() { + ldout(m_cct, 5) << dendl; + + auto ctx = new FunctionContext([this](int r) { + r = m_io_ctx.selfmanaged_snap_remove(m_snap_id); + handle_remove_snapshot(r); + }); + m_op_work_queue->queue(ctx, 0); +} + +template +void ValidatePoolRequest::handle_remove_snapshot(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + + if (r < 0) { + // not a fatal error + lderr(m_cct) << "failed to remove validation snapshot: " << cpp_strerror(r) + << dendl; + } + + if (m_ret_val < 0) { + finish(m_ret_val); + return; + } + + overwrite_rbd_info(); +} + +template +void ValidatePoolRequest::overwrite_rbd_info() { + ldout(m_cct, 5) << dendl; + + bufferlist bl; + bl.append(OVERWRITE_VALIDATED); + + librados::ObjectWriteOperation op; + op.write(0, bl); + + auto comp = create_rados_callback< + ValidatePoolRequest, + &ValidatePoolRequest::handle_overwrite_rbd_info>(this); + int r = m_io_ctx.aio_operate(RBD_INFO, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void ValidatePoolRequest::handle_overwrite_rbd_info(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + + if (r == -EOPNOTSUPP) { + lderr(m_cct) << "pool missing required overwrite support" << dendl; + finish(-EINVAL); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to validate overwrite support: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void ValidatePoolRequest::finish(int r) { + ldout(m_cct, 5) << "r=" << r << dendl; + m_on_finish->complete(r); + delete this; +} + +} // namespace image +} // namespace librbd + +template class librbd::image::ValidatePoolRequest; diff --git a/src/librbd/image/ValidatePoolRequest.h b/src/librbd/image/ValidatePoolRequest.h new file mode 100644 index 00000000000..8ea6e4a059d --- /dev/null +++ b/src/librbd/image/ValidatePoolRequest.h @@ -0,0 +1,96 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IMAGE_VALIDATE_POOL_REQUEST_H +#define CEPH_LIBRBD_IMAGE_VALIDATE_POOL_REQUEST_H + +#include "include/rados/librados.hpp" +#include "include/buffer.h" + +class CephContext; +class Context; +class ContextWQ; + +namespace librbd { + +struct ImageCtx; + +namespace image { + +template +class ValidatePoolRequest { +public: + static ValidatePoolRequest* create(librados::IoCtx& io_ctx, + ContextWQ *op_work_queue, + Context *on_finish) { + return new ValidatePoolRequest(io_ctx, op_work_queue, on_finish); + } + + ValidatePoolRequest(librados::IoCtx& io_ctx, ContextWQ *op_work_queue, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v (overwrites validated) + * READ RBD INFO . . . . . . . . . + * | . . + * | . (snapshots validated) . + * | . . . . . . . . . . . + * v . . + * CREATE SNAPSHOT . . + * | . . + * v . . + * WRITE RBD INFO . . + * | . . + * v . . + * REMOVE SNAPSHOT . . + * | . . + * v . . + * OVERWRITE RBD INFO < . . . . + * | . + * v . + * < . . . . . . . . . .` + * + * @endverbatim + */ + + librados::IoCtx m_io_ctx; + CephContext* m_cct; + ContextWQ* m_op_work_queue; + Context* m_on_finish; + + int m_ret_val = 0; + bufferlist m_out_bl; + uint64_t m_snap_id = 0; + + void read_rbd_info(); + void handle_read_rbd_info(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void write_rbd_info(); + void handle_write_rbd_info(int r); + + void remove_snapshot(); + void handle_remove_snapshot(int r); + + void overwrite_rbd_info(); + void handle_overwrite_rbd_info(int r); + + void finish(int r); + +}; + +} // namespace image +} // namespace librbd + +extern template class librbd::image::ValidatePoolRequest; + +#endif // CEPH_LIBRBD_IMAGE_VALIDATE_POOL_REQUEST_H diff --git a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h index 39c0e1ecb9d..0ef52d3863f 100644 --- a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h +++ b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h @@ -27,7 +27,8 @@ public: return m_mock_client; } - TestIoCtxImpl *clone() override { + MOCK_METHOD0(clone, TestIoCtxImpl*()); + TestIoCtxImpl *do_clone() { TestIoCtxImpl *io_ctx_impl = new ::testing::NiceMock( m_mock_client, m_client, get_pool_id(), get_pool_name(), get_pool()); io_ctx_impl->set_snap_read(get_snap_read()); @@ -194,6 +195,7 @@ public: void default_to_parent() { using namespace ::testing; + ON_CALL(*this, clone()).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_clone)); ON_CALL(*this, aio_notify(_, _, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_aio_notify)); ON_CALL(*this, aio_watch(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_aio_watch)); ON_CALL(*this, aio_unwatch(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_aio_unwatch)); diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index 571341a67c4..1c51884a483 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -36,6 +36,7 @@ set(unittest_librbd_srcs exclusive_lock/test_mock_PreReleaseRequest.cc image/test_mock_RefreshRequest.cc image/test_mock_RemoveRequest.cc + image/test_mock_ValidatePoolRequest.cc io/test_mock_ImageRequest.cc io/test_mock_ImageRequestWQ.cc io/test_mock_ObjectRequest.cc diff --git a/src/test/librbd/image/test_mock_ValidatePoolRequest.cc b/src/test/librbd/image/test_mock_ValidatePoolRequest.cc new file mode 100644 index 00000000000..e40e50af0ad --- /dev/null +++ b/src/test/librbd/image/test_mock_ValidatePoolRequest.cc @@ -0,0 +1,229 @@ +// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "test/librbd/test_support.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(ImageCtx &image_ctx) : MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "librbd/image/ValidatePoolRequest.cc" + +namespace librbd { +namespace image { + +using ::testing::_; +using ::testing::DoDefault; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageValidatePoolRequest : public TestMockFixture { +public: + typedef ValidatePoolRequest MockValidatePoolRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + ASSERT_EQ(0, open_image(m_image_name, &image_ctx)); + } + + void expect_clone(librados::MockTestMemIoCtxImpl &mock_io_ctx) { + EXPECT_CALL(mock_io_ctx, clone()) + .WillOnce(Invoke([&mock_io_ctx]() { + mock_io_ctx.get(); + return &mock_io_ctx; + })); + } + + void expect_read_rbd_info(librados::MockTestMemIoCtxImpl &mock_io_ctx, + const std::string& data, int r) { + auto& expect = EXPECT_CALL(mock_io_ctx, read(StrEq(RBD_INFO), 0, 0, _)); + if (r < 0) { + expect.WillOnce(Return(r)); + } else { + expect.WillOnce(WithArg<3>(Invoke([data](bufferlist* bl) { + bl->append(data); + return 0; + }))); + } + } + + void expect_write_rbd_info(librados::MockTestMemIoCtxImpl &mock_io_ctx, + const std::string& data, int r) { + bufferlist bl; + bl.append(data); + + EXPECT_CALL(mock_io_ctx, write(StrEq(RBD_INFO), ContentsEqual(bl), + data.length(), 0, _)) + .WillOnce(Return(r)); + } + + void expect_allocate_snap_id(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + auto &expect = EXPECT_CALL(mock_io_ctx, selfmanaged_snap_create(_)); + if (r < 0) { + expect.WillOnce(Return(r)); + } else { + expect.WillOnce(DoDefault()); + } + } + + void expect_release_snap_id(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + auto &expect = EXPECT_CALL(mock_io_ctx, selfmanaged_snap_remove(_)); + if (r < 0) { + expect.WillOnce(Return(r)); + } else { + expect.WillOnce(DoDefault()); + } + } + + librbd::ImageCtx *image_ctx; +}; + +TEST_F(TestMockImageValidatePoolRequest, Success) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", -ENOENT); + expect_allocate_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "validate", 0); + expect_release_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "overwrite validated", 0); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, AlreadyValidated) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "overwrite validated", 0); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, SnapshotsValidated) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "validate", 0); + expect_write_rbd_info(mock_io_ctx, "overwrite validated", 0); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, ReadError) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", -EPERM); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, CreateSnapshotError) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", 0); + expect_allocate_snap_id(mock_io_ctx, -EPERM); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, WriteError) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", -ENOENT); + expect_allocate_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "validate", -EPERM); + expect_release_snap_id(mock_io_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, RemoveSnapshotError) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", -ENOENT); + expect_allocate_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "validate", 0); + expect_release_snap_id(mock_io_ctx, -EPERM); + expect_write_rbd_info(mock_io_ctx, "overwrite validated", 0); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageValidatePoolRequest, OverwriteError) { + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx)); + + InSequence seq; + expect_clone(mock_io_ctx); + expect_read_rbd_info(mock_io_ctx, "", -ENOENT); + expect_allocate_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "validate", 0); + expect_release_snap_id(mock_io_ctx, 0); + expect_write_rbd_info(mock_io_ctx, "overwrite validated", -EOPNOTSUPP); + + C_SaferCond ctx; + auto req = new MockValidatePoolRequest(m_ioctx, image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image +} // namespace librbd