From: Jason Dillaman Date: Tue, 16 Aug 2016 18:20:20 +0000 (-0400) Subject: librbd: exclusive lock now supports reacquiring a lost lock X-Git-Tag: v10.2.6~66^2~21 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ead7201874ed8617d86dc548798aa4bde7168fdb;p=ceph.git librbd: exclusive lock now supports reacquiring a lost lock Signed-off-by: Jason Dillaman (cherry picked from commit 66c605573f840c0db8b3630315ea50e9fc987509) --- diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc index 99d7b1d25a0..b6c75ea14aa 100644 --- a/src/librbd/ExclusiveLock.cc +++ b/src/librbd/ExclusiveLock.cc @@ -10,6 +10,7 @@ #include "librbd/ImageWatcher.h" #include "librbd/Utils.h" #include "librbd/exclusive_lock/AcquireRequest.h" +#include "librbd/exclusive_lock/ReacquireRequest.h" #include "librbd/exclusive_lock/ReleaseRequest.h" #include @@ -60,6 +61,7 @@ bool ExclusiveLock::is_lock_owner() const { switch (m_state) { case STATE_LOCKED: case STATE_POST_ACQUIRING: + case STATE_REACQUIRING: case STATE_PRE_RELEASING: case STATE_PRE_SHUTTING_DOWN: lock_owner = true; @@ -194,6 +196,30 @@ void ExclusiveLock::release_lock(Context *on_released) { on_released->complete(r); } +template +void ExclusiveLock::reacquire_lock(Context *on_reacquired) { + { + Mutex::Locker locker(m_lock); + assert(m_image_ctx.owner_lock.is_locked()); + + // ignore request if shutdown or not in a locked-related state + if (!is_shutdown() && + (m_state == STATE_LOCKED || + m_state == STATE_ACQUIRING || + m_state == STATE_POST_ACQUIRING || + m_state == STATE_WAITING_FOR_REGISTER || + m_state == STATE_WAITING_FOR_PEER)) { + ldout(m_image_ctx.cct, 10) << this << " " << __func__ << dendl; + execute_action(ACTION_REACQUIRE_LOCK, on_reacquired); + return; + } + } + + if (on_reacquired != nullptr) { + on_reacquired->complete(0); + } +} + template void ExclusiveLock::handle_watch_registered() { Mutex::Locker locker(m_lock); @@ -256,6 +282,7 @@ bool ExclusiveLock::is_transition_state() const { case STATE_WAITING_FOR_PEER: case STATE_WAITING_FOR_REGISTER: case STATE_POST_ACQUIRING: + case STATE_REACQUIRING: case STATE_PRE_RELEASING: case STATE_RELEASING: case STATE_PRE_SHUTTING_DOWN: @@ -309,6 +336,9 @@ void ExclusiveLock::execute_next_action() { case ACTION_REQUEST_LOCK: send_acquire_lock(); break; + case ACTION_REACQUIRE_LOCK: + send_reacquire_lock(); + break; case ACTION_RELEASE_LOCK: send_release_lock(); break; @@ -457,6 +487,93 @@ void ExclusiveLock::handle_acquire_lock(int r) { complete_active_action(next_state, r); } +template +void ExclusiveLock::send_reacquire_lock() { + assert(m_lock.is_locked()); + + CephContext *cct = m_image_ctx.cct; + if (m_state != STATE_LOCKED) { + complete_active_action(m_state, 0); + return; + } + + m_watch_handle = m_image_ctx.image_watcher->get_watch_handle(); + if (m_watch_handle == 0) { + // watch (re)failed while recovering + lderr(cct) << this << " " << __func__ << ": " + << "aborting reacquire due to invalid watch handle" << dendl; + complete_active_action(STATE_LOCKED, 0); + return; + } + + m_new_cookie = encode_lock_cookie(); + if (m_cookie == m_new_cookie) { + ldout(cct, 10) << this << " " << __func__ << ": " + << "skipping reacquire since cookie still valid" << dendl; + complete_active_action(STATE_LOCKED, 0); + return; + } + + ldout(cct, 10) << this << " " << __func__ << dendl; + m_state = STATE_REACQUIRING; + + using el = ExclusiveLock; + ReacquireRequest* req = ReacquireRequest::create( + m_image_ctx, m_cookie, m_new_cookie, + util::create_context_callback(this)); + req->send(); +} + +template +void ExclusiveLock::handle_reacquire_lock(int r) { + Mutex::Locker locker(m_lock); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + assert(m_state == STATE_REACQUIRING); + if (r < 0) { + if (r == -EOPNOTSUPP) { + ldout(cct, 10) << this << " " << __func__ << ": " + << "updating lock is not supported" << dendl; + } else { + lderr(cct) << this << " " << __func__ << ": " + << "failed to update lock cookie: " << cpp_strerror(r) + << dendl; + } + + if (!is_shutdown()) { + // queue a release and re-acquire of the lock since cookie cannot + // be updated on older OSDs + execute_action(ACTION_RELEASE_LOCK, nullptr); + + assert(!m_actions_contexts.empty()); + ActionContexts &action_contexts(m_actions_contexts.front()); + + // reacquire completes when the request lock completes + Contexts contexts; + std::swap(contexts, action_contexts.second); + if (contexts.empty()) { + execute_action(ACTION_REQUEST_LOCK, nullptr); + } else { + for (auto ctx : contexts) { + ctx = new FunctionContext([ctx, r](int acquire_ret_val) { + if (acquire_ret_val >= 0) { + acquire_ret_val = r; + } + ctx->complete(acquire_ret_val); + }); + execute_action(ACTION_REQUEST_LOCK, ctx); + } + } + } + } else { + m_cookie = m_new_cookie; + } + + complete_active_action(STATE_LOCKED, 0); +} + template void ExclusiveLock::send_release_lock() { assert(m_lock.is_locked()); diff --git a/src/librbd/ExclusiveLock.h b/src/librbd/ExclusiveLock.h index 61fa019b64c..5ba17aed371 100644 --- a/src/librbd/ExclusiveLock.h +++ b/src/librbd/ExclusiveLock.h @@ -41,6 +41,8 @@ public: void request_lock(Context *on_locked); void release_lock(Context *on_released); + void reacquire_lock(Context *on_reacquired = nullptr); + void handle_watch_registered(); void handle_peer_notification(); @@ -51,6 +53,8 @@ public: private: /** + * @verbatim + * * * * > WAITING_FOR_REGISTER --------\ * | * (watch not registered) | * | * | @@ -69,11 +73,21 @@ private: * | (release_lock) v * PRE_RELEASING <------------------------ LOCKED * + * + * | + * v + * REACQUIRING -------------------------------------> + * . ^ + * . | + * . . . > ---> ---/ + * * * | * | * v * PRE_SHUTTING_DOWN ---> SHUTTING_DOWN ---> SHUTDOWN ---> + * + * @endverbatim */ enum State { STATE_UNINITIALIZED, @@ -84,16 +98,18 @@ private: STATE_POST_ACQUIRING, STATE_WAITING_FOR_PEER, STATE_WAITING_FOR_REGISTER, + STATE_REACQUIRING, STATE_PRE_RELEASING, STATE_RELEASING, STATE_PRE_SHUTTING_DOWN, STATE_SHUTTING_DOWN, - STATE_SHUTDOWN, + STATE_SHUTDOWN }; enum Action { ACTION_TRY_LOCK, ACTION_REQUEST_LOCK, + ACTION_REACQUIRE_LOCK, ACTION_RELEASE_LOCK, ACTION_SHUT_DOWN }; @@ -131,6 +147,7 @@ private: mutable Mutex m_lock; State m_state; std::string m_cookie; + std::string m_new_cookie; uint64_t m_watch_handle; ActionsContexts m_actions_contexts; @@ -157,6 +174,9 @@ private: void handle_acquiring_lock(int r); void handle_acquire_lock(int r); + void send_reacquire_lock(); + void handle_reacquire_lock(int r); + void send_release_lock(); void handle_releasing_lock(int r); void handle_release_lock(int r); diff --git a/src/test/librbd/test_mock_ExclusiveLock.cc b/src/test/librbd/test_mock_ExclusiveLock.cc index a2003a5d0a7..a0a87a1ccdd 100644 --- a/src/test/librbd/test_mock_ExclusiveLock.cc +++ b/src/test/librbd/test_mock_ExclusiveLock.cc @@ -6,6 +6,7 @@ #include "test/librbd/mock/MockImageCtx.h" #include "librbd/ExclusiveLock.h" #include "librbd/exclusive_lock/AcquireRequest.h" +#include "librbd/exclusive_lock/ReacquireRequest.h" #include "librbd/exclusive_lock/ReleaseRequest.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -53,6 +54,18 @@ struct AcquireRequest : public BaseRequest +struct ReacquireRequest : public BaseRequest > { + static ReacquireRequest* create(MockExclusiveLockImageCtx &image_ctx, + const std::string &cookie, + const std::string &new_cookie, + Context *on_finish) { + return BaseRequest::create(image_ctx, cookie, nullptr, on_finish); + } + + MOCK_METHOD0(send, void()); +}; + template <> struct ReleaseRequest : public BaseRequest > { MOCK_METHOD0(send, void()); @@ -83,11 +96,13 @@ class TestMockExclusiveLock : public TestMockFixture { public: typedef ExclusiveLock MockExclusiveLock; typedef exclusive_lock::AcquireRequest MockAcquireRequest; + typedef exclusive_lock::ReacquireRequest MockReacquireRequest; typedef exclusive_lock::ReleaseRequest MockReleaseRequest; - void expect_get_watch_handle(MockExclusiveLockImageCtx &mock_image_ctx) { + void expect_get_watch_handle(MockExclusiveLockImageCtx &mock_image_ctx, + uint64_t watch_handle = 1234567890) { EXPECT_CALL(*mock_image_ctx.image_watcher, get_watch_handle()) - .WillRepeatedly(Return(1234567890)); + .WillRepeatedly(Return(watch_handle)); } void expect_set_require_lock_on_read(MockExclusiveLockImageCtx &mock_image_ctx) { @@ -138,6 +153,14 @@ public: } } + void expect_reacquire_lock(MockExclusiveLockImageCtx &mock_image_ctx, + MockReacquireRequest &mock_reacquire_request, + int r) { + expect_get_watch_handle(mock_image_ctx, 98765); + EXPECT_CALL(mock_reacquire_request, send()) + .WillOnce(FinishRequest(&mock_reacquire_request, r, &mock_image_ctx)); + } + void expect_notify_request_lock(MockExclusiveLockImageCtx &mock_image_ctx, MockExclusiveLock &mock_exclusive_lock) { EXPECT_CALL(*mock_image_ctx.image_watcher, notify_request_lock()) @@ -176,7 +199,7 @@ public: } int when_try_lock(MockExclusiveLockImageCtx &mock_image_ctx, - MockExclusiveLock &exclusive_lock) { + MockExclusiveLock &exclusive_lock) { C_SaferCond ctx; { RWLock::WLocker owner_locker(mock_image_ctx.owner_lock); @@ -185,7 +208,7 @@ public: return ctx.wait(); } int when_request_lock(MockExclusiveLockImageCtx &mock_image_ctx, - MockExclusiveLock &exclusive_lock) { + MockExclusiveLock &exclusive_lock) { C_SaferCond ctx; { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); @@ -194,7 +217,7 @@ public: return ctx.wait(); } int when_release_lock(MockExclusiveLockImageCtx &mock_image_ctx, - MockExclusiveLock &exclusive_lock) { + MockExclusiveLock &exclusive_lock) { C_SaferCond ctx; { RWLock::WLocker owner_locker(mock_image_ctx.owner_lock); @@ -660,5 +683,80 @@ TEST_F(TestMockExclusiveLock, RequestLockWatchNotRegistered) { ASSERT_FALSE(is_lock_owner(mock_image_ctx, exclusive_lock)); } +TEST_F(TestMockExclusiveLock, ReacquireLock) { + REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockExclusiveLockImageCtx mock_image_ctx(*ictx); + MockExclusiveLock exclusive_lock(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_block_writes(mock_image_ctx); + ASSERT_EQ(0, when_init(mock_image_ctx, exclusive_lock)); + + MockAcquireRequest request_lock_acquire; + expect_acquire_lock(mock_image_ctx, request_lock_acquire, 0); + ASSERT_EQ(0, when_request_lock(mock_image_ctx, exclusive_lock)); + ASSERT_TRUE(is_lock_owner(mock_image_ctx, exclusive_lock)); + + MockReacquireRequest mock_reacquire_request; + C_SaferCond reacquire_ctx; + expect_reacquire_lock(mock_image_ctx, mock_reacquire_request, 0); + { + RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); + exclusive_lock.reacquire_lock(&reacquire_ctx); + } + ASSERT_EQ(0, reacquire_ctx.wait()); + + MockReleaseRequest shutdown_release; + expect_release_lock(mock_image_ctx, shutdown_release, 0, true); + ASSERT_EQ(0, when_shut_down(mock_image_ctx, exclusive_lock)); + ASSERT_FALSE(is_lock_owner(mock_image_ctx, exclusive_lock)); +} + +TEST_F(TestMockExclusiveLock, ReacquireLockError) { + REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockExclusiveLockImageCtx mock_image_ctx(*ictx); + MockExclusiveLock exclusive_lock(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + expect_block_writes(mock_image_ctx); + ASSERT_EQ(0, when_init(mock_image_ctx, exclusive_lock)); + + MockAcquireRequest request_lock_acquire; + expect_acquire_lock(mock_image_ctx, request_lock_acquire, 0); + ASSERT_EQ(0, when_request_lock(mock_image_ctx, exclusive_lock)); + ASSERT_TRUE(is_lock_owner(mock_image_ctx, exclusive_lock)); + + MockReacquireRequest mock_reacquire_request; + C_SaferCond reacquire_ctx; + expect_reacquire_lock(mock_image_ctx, mock_reacquire_request, -EOPNOTSUPP); + + MockReleaseRequest reacquire_lock_release; + expect_release_lock(mock_image_ctx, reacquire_lock_release, 0, false); + + MockAcquireRequest reacquire_lock_acquire; + expect_acquire_lock(mock_image_ctx, reacquire_lock_acquire, 0); + + { + RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); + exclusive_lock.reacquire_lock(&reacquire_ctx); + } + ASSERT_EQ(-EOPNOTSUPP, reacquire_ctx.wait()); + + MockReleaseRequest shutdown_release; + expect_release_lock(mock_image_ctx, shutdown_release, 0, true); + ASSERT_EQ(0, when_shut_down(mock_image_ctx, exclusive_lock)); + ASSERT_FALSE(is_lock_owner(mock_image_ctx, exclusive_lock)); +} + } // namespace librbd