#include "librbd/ObjectMap.h"
#include "librbd/Utils.h"
#include "librbd/exclusive_lock/ImageDispatch.h"
+#include "librbd/io/AioCompletion.h"
+#include "librbd/io/ImageDispatchSpec.h"
#include "librbd/io/ImageDispatcherInterface.h"
#include "librbd/io/ObjectDispatcherInterface.h"
+#include "librbd/io/Types.h"
#define dout_subsys ceph_subsys_rbd
#undef dout_prefix
template <typename I>
void PreReleaseRequest<I>::send() {
- send_prepare_lock();
-}
-
-template <typename I>
-void PreReleaseRequest<I>::send_prepare_lock() {
- if (m_shutting_down) {
- send_cancel_op_requests();
- return;
- }
-
- CephContext *cct = m_image_ctx.cct;
- ldout(cct, 10) << dendl;
-
- // release the lock if the image is not busy performing other actions
- Context *ctx = create_context_callback<
- PreReleaseRequest<I>, &PreReleaseRequest<I>::handle_prepare_lock>(this);
- m_image_ctx.state->prepare_lock(ctx);
-}
-
-template <typename I>
-void PreReleaseRequest<I>::handle_prepare_lock(int r) {
- CephContext *cct = m_image_ctx.cct;
- ldout(cct, 10) << "r=" << r << dendl;
-
send_cancel_op_requests();
}
ceph_assert(r == 0);
- send_block_writes();
+ send_set_require_lock();
}
template <typename I>
-void PreReleaseRequest<I>::send_block_writes() {
+void PreReleaseRequest<I>::send_set_require_lock() {
CephContext *cct = m_image_ctx.cct;
ldout(cct, 10) << dendl;
using klass = PreReleaseRequest<I>;
Context *ctx = create_context_callback<
- klass, &klass::handle_block_writes>(this);
+ klass, &klass::handle_set_require_lock>(this);
// setting the lock as required will automatically cause the IO
// queue to re-request the lock if any IO is queued
}
template <typename I>
-void PreReleaseRequest<I>::handle_block_writes(int r) {
+void PreReleaseRequest<I>::handle_set_require_lock(int r) {
CephContext *cct = m_image_ctx.cct;
ldout(cct, 10) << "r=" << r << dendl;
- if (r == -EBLOCKLISTED) {
- // allow clean shut down if blocklisted
- lderr(cct) << "failed to block writes because client is blocklisted"
- << dendl;
- } else if (r < 0) {
- lderr(cct) << "failed to block writes: " << cpp_strerror(r) << dendl;
- m_image_dispatch->unset_require_lock(io::DIRECTION_BOTH);
- save_result(r);
- finish();
- return;
+ if (r < 0) {
+ // IOs are still flushed regardless of the error
+ lderr(cct) << "failed to set lock: " << cpp_strerror(r) << dendl;
}
send_wait_for_ops();
CephContext *cct = m_image_ctx.cct;
ldout(cct, 10) << dendl;
+ send_prepare_lock();
+}
+
+template <typename I>
+void PreReleaseRequest<I>::send_prepare_lock() {
+ if (m_shutting_down) {
+ send_shut_down_image_cache();
+ return;
+ }
+
+ CephContext *cct = m_image_ctx.cct;
+ ldout(cct, 10) << dendl;
+
+ // release the lock if the image is not busy performing other actions
+ Context *ctx = create_context_callback<
+ PreReleaseRequest<I>, &PreReleaseRequest<I>::handle_prepare_lock>(this);
+ m_image_ctx.state->prepare_lock(ctx);
+}
+
+template <typename I>
+void PreReleaseRequest<I>::handle_prepare_lock(int r) {
+ CephContext *cct = m_image_ctx.cct;
+ ldout(cct, 10) << "r=" << r << dendl;
+
send_shut_down_image_cache();
}
return;
}
+ send_flush_io();
+}
+
+template <typename I>
+void PreReleaseRequest<I>::send_flush_io() {
+ CephContext *cct = m_image_ctx.cct;
+ ldout(cct, 10) << dendl;
+
+ // ensure that all in-flight IO is flushed -- skipping the refresh layer
+ // since it should have been flushed when the lock was required and now
+ // refreshes are disabled / interlocked w/ this state machine.
+ auto ctx = create_context_callback<
+ PreReleaseRequest<I>, &PreReleaseRequest<I>::handle_flush_io>(this);
+ auto aio_comp = io::AioCompletion::create_and_start(
+ ctx, util::get_image_ctx(&m_image_ctx), librbd::io::AIO_TYPE_FLUSH);
+ auto req = io::ImageDispatchSpec::create_flush(
+ m_image_ctx, io::IMAGE_DISPATCH_LAYER_EXCLUSIVE_LOCK, aio_comp,
+ io::FLUSH_SOURCE_EXCLUSIVE_LOCK_SKIP_REFRESH, {});
+ req->send();
+}
+
+template <typename I>
+void PreReleaseRequest<I>::handle_flush_io(int r) {
+ CephContext *cct = m_image_ctx.cct;
+ ldout(cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(cct) << "failed to flush IO: " << cpp_strerror(r) << dendl;
+ }
+
send_flush_notifies();
}
* <start>
* |
* v
- * PREPARE_LOCK
- * |
- * v
* CANCEL_OP_REQUESTS
* |
* v
- * BLOCK_WRITES
+ * SET_REQUIRE_LOCK
* |
* v
* WAIT_FOR_OPS
* |
* v
+ * PREPARE_LOCK
+ * |
+ * v
* SHUT_DOWN_IMAGE_CACHE
* |
* v
* INVALIDATE_CACHE
* |
* v
+ * FLUSH_IO
+ * |
+ * v
* FLUSH_NOTIFIES . . . . . . . . . . . . . .
* | .
* v .
decltype(m_image_ctx.object_map) m_object_map = nullptr;
decltype(m_image_ctx.journal) m_journal = nullptr;
- void send_prepare_lock();
- void handle_prepare_lock(int r);
-
void send_cancel_op_requests();
void handle_cancel_op_requests(int r);
- void send_block_writes();
- void handle_block_writes(int r);
+ void send_set_require_lock();
+ void handle_set_require_lock(int r);
void send_wait_for_ops();
void handle_wait_for_ops(int r);
+ void send_prepare_lock();
+ void handle_prepare_lock(int r);
+
void send_shut_down_image_cache();
void handle_shut_down_image_cache(int r);
void send_invalidate_cache();
void handle_invalidate_cache(int r);
+ void send_flush_io();
+ void handle_flush_io(int r);
+
void send_flush_notifies();
void handle_flush_notifies(int r);
} // namespace pwl
} // namespace cache
+
+namespace util {
+
+inline ImageCtx* get_image_ctx(MockTestImageCtx* image_ctx) {
+ return image_ctx->image_ctx;
+}
+
+} // namespace util
} // namespace librbd
// template definitions
EXPECT_CALL(*mock_image_ctx.state, handle_prepare_lock_complete());
}
+ void expect_flush_io(MockTestImageCtx &mock_image_ctx, int r) {
+ EXPECT_CALL(*mock_image_ctx.io_image_dispatcher, send(_))
+ .WillOnce(Invoke([&mock_image_ctx, r](io::ImageDispatchSpec* spec) {
+ ASSERT_TRUE(boost::get<io::ImageDispatchSpec::Flush>(
+ &spec->request) != nullptr);
+ spec->dispatch_result = io::DISPATCH_RESULT_COMPLETE;
+ auto aio_comp = spec->aio_comp;
+ auto ctx = new LambdaContext([aio_comp](int r) {
+ if (r < 0) {
+ aio_comp->fail(r);
+ } else {
+ aio_comp->set_request_count(1);
+ aio_comp->add_request();
+ aio_comp->complete_request(r);
+ }
+ });
+ mock_image_ctx.image_ctx->op_work_queue->queue(ctx, r);
+ }));
+ }
+
AsyncOpTracker m_async_op_tracker;
};
InSequence seq;
- expect_prepare_lock(mock_image_ctx);
expect_cancel_op_requests(mock_image_ctx, 0);
MockImageDispatch mock_image_dispatch;
expect_set_require_lock(mock_image_ctx, mock_image_dispatch, false, 0);
+ expect_prepare_lock(mock_image_ctx);
+
cache::MockImageCache mock_image_cache;
mock_image_ctx.image_cache = &mock_image_cache;
MockShutdownRequest mock_shutdown_request;
expect_invalidate_cache(mock_image_ctx, 0);
+ expect_flush_io(mock_image_ctx, 0);
+
expect_flush_notifies(mock_image_ctx);
MockJournal mock_journal;
expect_op_work_queue(mock_image_ctx);
InSequence seq;
- expect_prepare_lock(mock_image_ctx);
expect_cancel_op_requests(mock_image_ctx, 0);
+ expect_prepare_lock(mock_image_ctx);
cache::MockImageCache mock_image_cache;
mock_image_ctx.image_cache = &mock_image_cache;
expect_invalidate_cache(mock_image_ctx, 0);
+ expect_flush_io(mock_image_ctx, 0);
+
expect_flush_notifies(mock_image_ctx);
MockObjectMap mock_object_map;
expect_invalidate_cache(mock_image_ctx, 0);
+ expect_flush_io(mock_image_ctx, 0);
+
expect_flush_notifies(mock_image_ctx);
C_SaferCond release_ctx;
expect_op_work_queue(mock_image_ctx);
InSequence seq;
- expect_prepare_lock(mock_image_ctx);
expect_cancel_op_requests(mock_image_ctx, 0);
MockImageDispatch mock_image_dispatch;
expect_set_require_lock(mock_image_ctx, mock_image_dispatch, false,
-EBLOCKLISTED);
+ expect_prepare_lock(mock_image_ctx);
cache::MockImageCache mock_image_cache;
mock_image_ctx.image_cache = &mock_image_cache;
expect_invalidate_cache(mock_image_ctx, -EBLOCKLISTED);
+ expect_flush_io(mock_image_ctx, -EBLOCKLISTED);
+
expect_flush_notifies(mock_image_ctx);
MockJournal mock_journal;
ASSERT_EQ(0, ctx.wait());
}
-TEST_F(TestMockExclusiveLockPreReleaseRequest, BlockWritesError) {
- REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
-
- librbd::ImageCtx *ictx;
- ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
- MockTestImageCtx mock_image_ctx(*ictx);
-
- expect_op_work_queue(mock_image_ctx);
-
- InSequence seq;
- expect_cancel_op_requests(mock_image_ctx, 0);
- MockImageDispatch mock_image_dispatch;
- expect_set_require_lock(mock_image_ctx, mock_image_dispatch, true, -EINVAL);
- expect_unset_require_lock(mock_image_dispatch);
-
- C_SaferCond ctx;
- MockPreReleaseRequest *req = MockPreReleaseRequest::create(
- mock_image_ctx, &mock_image_dispatch, true, m_async_op_tracker, &ctx);
- req->send();
- ASSERT_EQ(-EINVAL, ctx.wait());
-}
-
} // namespace exclusive_lock
} // namespace librbd
projects / ceph.git / commitdiff