From: Ricardo Dias Date: Tue, 25 Oct 2016 13:39:58 +0000 (+0100) Subject: rbd: Implementation of a generic managed-lock X-Git-Tag: v12.0.0~225^2~2^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0b47ea705ffb1c70a1ffb8a3ad29849bf3806bfe;p=ceph.git rbd: Implementation of a generic managed-lock Signed-off-by: Ricardo Dias --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 5474dd98403..ff58d68eb16 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -31,6 +31,10 @@ set(librbd_internal_srcs Watcher.cc watcher/Types.cc watcher/RewatchRequest.cc + managed_lock/AcquireRequest.cc + managed_lock/ReleaseRequest.cc + managed_lock/ReacquireRequest.cc + ManagedLock.cc exclusive_lock/AcquireRequest.cc exclusive_lock/AutomaticPolicy.cc exclusive_lock/ReacquireRequest.cc diff --git a/src/librbd/ManagedLock.cc b/src/librbd/ManagedLock.cc new file mode 100644 index 00000000000..0447cc24b48 --- /dev/null +++ b/src/librbd/ManagedLock.cc @@ -0,0 +1,680 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/ManagedLock.h" +#include "librbd/managed_lock/AcquireRequest.h" +#include "librbd/managed_lock/ReleaseRequest.h" +#include "librbd/managed_lock/ReacquireRequest.h" +#include "librbd/Watcher.h" +#include "librbd/ImageCtx.h" +#include "cls/lock/cls_lock_client.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "librbd/Utils.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::ManagedLock: " << this << " " << __func__ + +namespace librbd { + +using std::string; + +namespace { + +const std::string WATCHER_LOCK_COOKIE_PREFIX = "auto"; + +template +struct C_SendLockRequest : public Context { + R* request; + explicit C_SendLockRequest(R* request) : request(request) { + } + virtual void finish(int r) override { + request->send(); + } +}; + +} // anonymous namespace + +template +const std::string ManagedLock::WATCHER_LOCK_TAG("internal"); + +template +ManagedLock::ManagedLock(librados::IoCtx &ioctx, ContextWQ *work_queue, + const string& oid, Watcher *watcher) + : m_lock(util::unique_lock_name("librbd::ManagedLock::m_lock", this)), + m_state(STATE_UNLOCKED), + m_ioctx(ioctx), m_cct(reinterpret_cast(ioctx.cct())), + m_work_queue(work_queue), + m_oid(oid), + m_watcher(watcher) { +} + +template +ManagedLock::~ManagedLock() { + Mutex::Locker locker(m_lock); + assert(m_state == STATE_SHUTDOWN || m_state == STATE_UNLOCKED || + m_state == STATE_UNINITIALIZED); +} + +template +bool ManagedLock::is_lock_owner() const { + Mutex::Locker locker(m_lock); + + bool lock_owner; + + switch (m_state) { + case STATE_LOCKED: + case STATE_REACQUIRING: + case STATE_PRE_SHUTTING_DOWN: + case STATE_POST_ACQUIRING: + case STATE_PRE_RELEASING: + lock_owner = true; + break; + default: + lock_owner = false; + break; + } + + ldout(m_cct, 20) << "=" << lock_owner << dendl; + return lock_owner; +} + +template +void ManagedLock::shut_down(Context *on_shut_down) { + ldout(m_cct, 10) << dendl; + + Mutex::Locker locker(m_lock); + assert(!is_shutdown_locked()); + execute_action(ACTION_SHUT_DOWN, on_shut_down); +} + +template +void ManagedLock::acquire_lock(Context *on_acquired) { + int r = 0; + { + Mutex::Locker locker(m_lock); + if (is_shutdown_locked()) { + r = -ESHUTDOWN; + } else if (m_state != STATE_LOCKED || !m_actions_contexts.empty()) { + ldout(m_cct, 10) << dendl; + execute_action(ACTION_ACQUIRE_LOCK, on_acquired); + return; + } + } + + on_acquired->complete(r); +} + +template +void ManagedLock::try_acquire_lock(Context *on_acquired) { + int r = 0; + { + Mutex::Locker locker(m_lock); + if (is_shutdown_locked()) { + r = -ESHUTDOWN; + } else if (m_state != STATE_LOCKED || !m_actions_contexts.empty()) { + ldout(m_cct, 10) << dendl; + execute_action(ACTION_TRY_LOCK, on_acquired); + return; + } + } + + on_acquired->complete(r); +} + +template +void ManagedLock::release_lock(Context *on_released) { + int r = 0; + { + Mutex::Locker locker(m_lock); + if (is_shutdown_locked()) { + r = -ESHUTDOWN; + } else if (m_state != STATE_UNLOCKED || !m_actions_contexts.empty()) { + ldout(m_cct, 10) << dendl; + execute_action(ACTION_RELEASE_LOCK, on_released); + return; + } + } + + on_released->complete(r); +} + +template +void ManagedLock::reacquire_lock(Context *on_reacquired) { + { + Mutex::Locker locker(m_lock); + + if (m_state == STATE_WAITING_FOR_REGISTER) { + // restart the acquire lock process now that watch is valid + ldout(m_cct, 10) << ": " << "woke up waiting acquire" << dendl; + Action active_action = get_active_action(); + assert(active_action == ACTION_TRY_LOCK || + active_action == ACTION_ACQUIRE_LOCK); + execute_next_action(); + } else if (!is_shutdown_locked() && + (m_state == STATE_LOCKED || + m_state == STATE_ACQUIRING || + m_state == STATE_POST_ACQUIRING || + m_state == STATE_WAITING_FOR_LOCK)) { + // interlock the lock operation with other state ops + ldout(m_cct, 10) << dendl; + execute_action(ACTION_REACQUIRE_LOCK, on_reacquired); + return; + } + } + + // ignore request if shutdown or not in a locked-related state + if (on_reacquired != nullptr) { + on_reacquired->complete(0); + } +} + +template +void ManagedLock::shutdown_handler(int r, Context *on_finish) { + on_finish->complete(r); +} + +template +void ManagedLock::pre_acquire_lock_handler(Context *on_finish) { + on_finish->complete(0); +} + +template +void ManagedLock::post_acquire_lock_handler(int r, Context *on_finish) { + on_finish->complete(r); +} + +template +void ManagedLock::pre_release_lock_handler(bool shutting_down, + Context *on_finish) { + { + Mutex::Locker locker(m_lock); + m_state = shutting_down ? STATE_SHUTTING_DOWN : STATE_RELEASING; + } + on_finish->complete(0); +} + +template +void ManagedLock::post_release_lock_handler(bool shutting_down, int r, + Context *on_finish) { + on_finish->complete(r); +} + +template +void ManagedLock::assert_locked(librados::ObjectWriteOperation *op, + ClsLockType type) { + Mutex::Locker locker(m_lock); + rados::cls::lock::assert_locked(op, RBD_LOCK_NAME, type, m_cookie, + WATCHER_LOCK_TAG); +} + +template +bool ManagedLock::decode_lock_cookie(const std::string &tag, + uint64_t *handle) { + std::string prefix; + std::istringstream ss(tag); + if (!(ss >> prefix >> *handle) || prefix != WATCHER_LOCK_COOKIE_PREFIX) { + return false; + } + return true; +} + +template +string ManagedLock::encode_lock_cookie(uint64_t watch_handle) { + assert(watch_handle != 0); + std::ostringstream ss; + ss << WATCHER_LOCK_COOKIE_PREFIX << " " << watch_handle; + return ss.str(); +} + +template +bool ManagedLock::is_transition_state() const { + switch (m_state) { + case STATE_ACQUIRING: + case STATE_WAITING_FOR_REGISTER: + case STATE_REACQUIRING: + case STATE_RELEASING: + case STATE_PRE_SHUTTING_DOWN: + case STATE_SHUTTING_DOWN: + case STATE_INITIALIZING: + case STATE_WAITING_FOR_LOCK: + case STATE_POST_ACQUIRING: + case STATE_PRE_RELEASING: + return true; + case STATE_UNLOCKED: + case STATE_LOCKED: + case STATE_SHUTDOWN: + case STATE_UNINITIALIZED: + break; + } + return false; +} + +template +void ManagedLock::append_context(Action action, Context *ctx) { + assert(m_lock.is_locked()); + + for (auto &action_ctxs : m_actions_contexts) { + if (action == action_ctxs.first) { + if (ctx != nullptr) { + action_ctxs.second.push_back(ctx); + } + return; + } + } + + Contexts contexts; + if (ctx != nullptr) { + contexts.push_back(ctx); + } + m_actions_contexts.push_back({action, std::move(contexts)}); +} + +template +void ManagedLock::execute_action(Action action, Context *ctx) { + assert(m_lock.is_locked()); + + append_context(action, ctx); + if (!is_transition_state()) { + execute_next_action(); + } +} + +template +void ManagedLock::execute_next_action() { + assert(m_lock.is_locked()); + assert(!m_actions_contexts.empty()); + switch (get_active_action()) { + case ACTION_ACQUIRE_LOCK: + case ACTION_TRY_LOCK: + send_acquire_lock(); + break; + case ACTION_REACQUIRE_LOCK: + send_reacquire_lock(); + break; + case ACTION_RELEASE_LOCK: + send_release_lock(); + break; + case ACTION_SHUT_DOWN: + send_shutdown(); + break; + default: + assert(false); + break; + } +} + +template +typename ManagedLock::Action ManagedLock::get_active_action() const { + assert(m_lock.is_locked()); + assert(!m_actions_contexts.empty()); + return m_actions_contexts.front().first; +} + +template +void ManagedLock::complete_active_action(State next_state, int r) { + assert(m_lock.is_locked()); + assert(!m_actions_contexts.empty()); + + ActionContexts action_contexts(std::move(m_actions_contexts.front())); + m_actions_contexts.pop_front(); + m_state = next_state; + + m_lock.Unlock(); + for (auto ctx : action_contexts.second) { + ctx->complete(r); + } + m_lock.Lock(); + + if (!is_transition_state() && !m_actions_contexts.empty()) { + execute_next_action(); + } +} + +template +bool ManagedLock::is_shutdown_locked() const { + assert(m_lock.is_locked()); + + return ((m_state == STATE_SHUTDOWN) || + (!m_actions_contexts.empty() && + m_actions_contexts.back().first == ACTION_SHUT_DOWN)); +} + +template +void ManagedLock::send_acquire_lock() { + assert(m_lock.is_locked()); + if (m_state == STATE_LOCKED) { + complete_active_action(STATE_LOCKED, 0); + return; + } + + ldout(m_cct, 10) << dendl; + m_state = STATE_ACQUIRING; + + uint64_t watch_handle = m_watcher->get_watch_handle(); + if (watch_handle == 0) { + lderr(m_cct) << "watcher not registered - delaying request" << dendl; + m_state = STATE_WAITING_FOR_REGISTER; + return; + } + m_cookie = ManagedLock::encode_lock_cookie(watch_handle); + + m_work_queue->queue(new FunctionContext([this](int r) { + pre_acquire_lock_handler(util::create_context_callback< + ManagedLock, &ManagedLock::handle_pre_acquire_lock>(this)); + })); +} + +template +void ManagedLock::handle_pre_acquire_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + if (r < 0) { + handle_acquire_lock(r); + return; + } + + using managed_lock::AcquireRequest; + AcquireRequest* req = AcquireRequest::create(m_ioctx, m_watcher, + m_work_queue, m_oid, m_cookie, + util::create_context_callback< + ManagedLock, &ManagedLock::handle_acquire_lock>(this)); + m_work_queue->queue(new C_SendLockRequest>(req), 0); +} + +template +void ManagedLock::handle_acquire_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + if (r == -EBUSY || r == -EAGAIN) { + ldout(m_cct, 5) << ": unable to acquire exclusive lock" << dendl; + } else if (r < 0) { + lderr(m_cct) << ": failed to acquire exclusive lock:" << cpp_strerror(r) + << dendl; + } else { + ldout(m_cct, 5) << ": successfully acquired exclusive lock" << dendl; + } + + m_post_next_state = (r < 0 ? STATE_UNLOCKED : STATE_LOCKED); + + m_work_queue->queue(new FunctionContext([this, r](int ret) { + post_acquire_lock_handler(r, util::create_context_callback< + ManagedLock, &ManagedLock::handle_post_acquire_lock>(this)); + })); +} + +template +void ManagedLock::handle_post_acquire_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + Mutex::Locker locker(m_lock); + + if (r < 0 && m_post_next_state == STATE_LOCKED) { + // release_lock without calling pre and post handlers + revert_to_unlock_state(r); + } else { + complete_active_action(m_post_next_state, r); + } +} + +template +void ManagedLock::revert_to_unlock_state(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + using managed_lock::ReleaseRequest; + ReleaseRequest* req = ReleaseRequest::create(m_ioctx, m_watcher, + m_work_queue, m_oid, m_cookie, + new FunctionContext([this, r](int ret) { + Mutex::Locker locker(m_lock); + assert(ret == 0); + complete_active_action(STATE_UNLOCKED, r); + })); + m_work_queue->queue(new C_SendLockRequest>(req)); +} + +template +void ManagedLock::send_reacquire_lock() { + assert(m_lock.is_locked()); + + if (m_state != STATE_LOCKED) { + complete_active_action(m_state, 0); + return; + } + + uint64_t watch_handle = m_watcher->get_watch_handle(); + if (watch_handle == 0) { + // watch (re)failed while recovering + lderr(m_cct) << ": aborting reacquire due to invalid watch handle" + << dendl; + complete_active_action(STATE_LOCKED, 0); + return; + } + + m_new_cookie = ManagedLock::encode_lock_cookie(watch_handle); + if (m_cookie == m_new_cookie) { + ldout(m_cct, 10) << ": skipping reacquire since cookie still valid" + << dendl; + complete_active_action(STATE_LOCKED, 0); + return; + } + + ldout(m_cct, 10) << dendl; + m_state = STATE_REACQUIRING; + + using managed_lock::ReacquireRequest; + ReacquireRequest* req = ReacquireRequest::create(m_ioctx, m_oid, + m_cookie, m_new_cookie, + util::create_context_callback< + ManagedLock, &ManagedLock::handle_reacquire_lock>(this)); + m_work_queue->queue(new C_SendLockRequest>(req)); +} + +template +void ManagedLock::handle_reacquire_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + Mutex::Locker locker(m_lock); + assert(m_state == STATE_REACQUIRING); + + if (r < 0) { + if (r == -EOPNOTSUPP) { + ldout(m_cct, 10) << ": updating lock is not supported" << dendl; + } else { + lderr(m_cct) << ": failed to update lock cookie: " << cpp_strerror(r) + << dendl; + } + + if (!is_shutdown_locked()) { + // queue a release and re-acquire of the lock since cookie cannot + // be updated on older OSDs + execute_action(ACTION_RELEASE_LOCK, nullptr); + + assert(!m_actions_contexts.empty()); + ActionContexts &action_contexts(m_actions_contexts.front()); + + // reacquire completes when the request lock completes + Contexts contexts; + std::swap(contexts, action_contexts.second); + if (contexts.empty()) { + execute_action(ACTION_ACQUIRE_LOCK, nullptr); + } else { + for (auto ctx : contexts) { + ctx = new FunctionContext([ctx, r](int acquire_ret_val) { + if (acquire_ret_val >= 0) { + acquire_ret_val = r; + } + ctx->complete(acquire_ret_val); + }); + execute_action(ACTION_ACQUIRE_LOCK, ctx); + } + } + } + } else { + m_cookie = m_new_cookie; + } + + complete_active_action(STATE_LOCKED, r); +} + +template +void ManagedLock::send_release_lock() { + assert(m_lock.is_locked()); + if (m_state == STATE_UNLOCKED) { + complete_active_action(STATE_UNLOCKED, 0); + return; + } + + ldout(m_cct, 10) << dendl; + m_state = STATE_PRE_RELEASING; + + m_work_queue->queue(new FunctionContext([this](int r) { + pre_release_lock_handler(false, util::create_context_callback< + ManagedLock, &ManagedLock::handle_pre_release_lock>(this)); + })); +} + +template +void ManagedLock::handle_pre_release_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + if (r < 0) { + handle_release_lock(r); + return; + } + + using managed_lock::ReleaseRequest; + ReleaseRequest* req = ReleaseRequest::create(m_ioctx, m_watcher, + m_work_queue, m_oid, m_cookie, + util::create_context_callback< + ManagedLock, &ManagedLock::handle_release_lock>(this)); + m_work_queue->queue(new C_SendLockRequest>(req), 0); +} + +template +void ManagedLock::handle_release_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + Mutex::Locker locker(m_lock); + assert(m_state == STATE_RELEASING); + + if (r >= 0) { + m_cookie = ""; + } + + m_post_next_state = r < 0 ? STATE_LOCKED : STATE_UNLOCKED; + + m_work_queue->queue(new FunctionContext([this, r](int ret) { + post_release_lock_handler(false, r, util::create_context_callback< + ManagedLock, &ManagedLock::handle_post_release_lock>(this)); + })); +} + +template +void ManagedLock::handle_post_release_lock(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + Mutex::Locker locker(m_lock); + complete_active_action(m_post_next_state, r); +} + +template +void ManagedLock::send_shutdown() { + ldout(m_cct, 10) << dendl; + assert(m_lock.is_locked()); + if (m_state == STATE_UNLOCKED) { + m_state = STATE_SHUTTING_DOWN; + m_work_queue->queue(new FunctionContext([this](int r) { + shutdown_handler(r, util::create_context_callback< + ManagedLock, &ManagedLock::handle_shutdown>(this)); + })); + return; + } + + ldout(m_cct, 10) << dendl; + assert(m_state == STATE_LOCKED); + m_state = STATE_PRE_SHUTTING_DOWN; + + m_lock.Unlock(); + m_work_queue->queue(new C_ShutDownRelease(this), 0); + m_lock.Lock(); +} + +template +void ManagedLock::handle_shutdown(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + complete_shutdown(r); +} + +template +void ManagedLock::send_shutdown_release() { + ldout(m_cct, 10) << dendl; + + Mutex::Locker locker(m_lock); + + m_work_queue->queue(new FunctionContext([this](int r) { + pre_release_lock_handler(true, util::create_context_callback< + ManagedLock, &ManagedLock::handle_shutdown_pre_release>(this)); + })); +} + +template +void ManagedLock::handle_shutdown_pre_release(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + std::string cookie; + { + Mutex::Locker locker(m_lock); + cookie = m_cookie; + } + + using managed_lock::ReleaseRequest; + ReleaseRequest* req = ReleaseRequest::create(m_ioctx, m_watcher, + m_work_queue, m_oid, cookie, + new FunctionContext([this](int r) { + post_release_lock_handler(true, r, util::create_context_callback< + ManagedLock, &ManagedLock::handle_shutdown_post_release>(this)); + })); + req->send(); + +} + +template +void ManagedLock::handle_shutdown_post_release(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + complete_shutdown(r); +} + +template +void ManagedLock::complete_shutdown(int r) { + ldout(m_cct, 10) << ": r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to shut down lock: " << cpp_strerror(r) + << dendl; + } + + ActionContexts action_contexts; + { + Mutex::Locker locker(m_lock); + assert(m_lock.is_locked()); + assert(m_actions_contexts.size() == 1); + + action_contexts = std::move(m_actions_contexts.front()); + m_actions_contexts.pop_front(); + m_state = STATE_SHUTDOWN; + } + + // expect to be destroyed after firing callback + for (auto ctx : action_contexts.second) { + ctx->complete(r); + } +} + +} // namespace librbd + +template class librbd::ManagedLock; diff --git a/src/librbd/ManagedLock.h b/src/librbd/ManagedLock.h new file mode 100644 index 00000000000..693ce623af7 --- /dev/null +++ b/src/librbd/ManagedLock.h @@ -0,0 +1,196 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MANAGED_LOCK_H +#define CEPH_LIBRBD_MANAGED_LOCK_H + +#include "include/int_types.h" +#include "include/Context.h" +#include "include/rados/librados.hpp" +#include "cls/lock/cls_lock_types.h" +#include "librbd/watcher/Types.h" +#include "common/Mutex.h" +#include +#include +#include + +class ContextWQ; + +namespace librbd { + +template +class ManagedLock { +private: + typedef watcher::Traits TypeTraits; + typedef typename TypeTraits::Watcher Watcher; + +public: + static const std::string WATCHER_LOCK_TAG; + + static ManagedLock *create(librados::IoCtx& ioctx, ContextWQ *work_queue, + const std::string& oid, Watcher *watcher) { + return new ManagedLock(ioctx, work_queue, oid, watcher); + } + + ManagedLock(librados::IoCtx& ioctx, ContextWQ *work_queue, + const std::string& oid, Watcher *watcher); + virtual ~ManagedLock(); + + bool is_lock_owner() const; + + void shut_down(Context *on_shutdown); + void acquire_lock(Context *on_acquired); + void try_acquire_lock(Context *on_acquired); + void release_lock(Context *on_released); + void reacquire_lock(Context *on_reacquired = nullptr); + + void assert_locked(librados::ObjectWriteOperation *op, ClsLockType type); + + bool is_shutdown() const { + Mutex::Locker l(m_lock); + return is_shutdown_locked(); + } + + bool is_locked_state() const { + return m_state == STATE_LOCKED; + } + + static bool decode_lock_cookie(const std::string &tag, uint64_t *handle); + +protected: + + /** + * @verbatim + * + * + * | + * | + * v (acquire_lock) + * UNLOCKED -----------------------------------------> ACQUIRING + * ^ | + * | | + * RELEASING | + * | | + * | | + * | (release_lock) v + * PRE_RELEASING <----------------------------------------- LOCKED + * + * + * | + * v + * REACQUIRING -------------------------------------> + * . ^ + * . | + * . . . > ---> ---/ + * + * + * | + * | + * v + * PRE_SHUTTING_DOWN ---> SHUTTING_DOWN ---> SHUTDOWN ---> + * + * @endverbatim + */ + enum State { + STATE_UNINITIALIZED, + STATE_INITIALIZING, + STATE_UNLOCKED, + STATE_LOCKED, + STATE_ACQUIRING, + STATE_POST_ACQUIRING, + STATE_WAITING_FOR_REGISTER, + STATE_WAITING_FOR_LOCK, + STATE_REACQUIRING, + STATE_PRE_RELEASING, + STATE_RELEASING, + STATE_PRE_SHUTTING_DOWN, + STATE_SHUTTING_DOWN, + STATE_SHUTDOWN, + }; + + enum Action { + ACTION_TRY_LOCK, + ACTION_ACQUIRE_LOCK, + ACTION_REACQUIRE_LOCK, + ACTION_RELEASE_LOCK, + ACTION_SHUT_DOWN + }; + + mutable Mutex m_lock; + State m_state; + + virtual void shutdown_handler(int r, Context *on_finish); + virtual void pre_acquire_lock_handler(Context *on_finish); + virtual void post_acquire_lock_handler(int r, Context *on_finish); + virtual void pre_release_lock_handler(bool shutting_down, + Context *on_finish); + virtual void post_release_lock_handler(bool shutting_down, int r, + Context *on_finish); + + Action get_active_action() const; + bool is_shutdown_locked() const; + void execute_next_action(); + +private: + typedef std::list Contexts; + typedef std::pair ActionContexts; + typedef std::list ActionsContexts; + + struct C_ShutDownRelease : public Context { + ManagedLock *lock; + C_ShutDownRelease(ManagedLock *lock) + : lock(lock) { + } + virtual void finish(int r) override { + lock->send_shutdown_release(); + } + }; + + librados::IoCtx& m_ioctx; + CephContext *m_cct; + ContextWQ *m_work_queue; + std::string m_oid; + Watcher *m_watcher; + + std::string m_cookie; + std::string m_new_cookie; + + State m_post_next_state; + + ActionsContexts m_actions_contexts; + + static std::string encode_lock_cookie(uint64_t watch_handle); + + bool is_transition_state() const; + + void append_context(Action action, Context *ctx); + void execute_action(Action action, Context *ctx); + + void complete_active_action(State next_state, int r); + + + void send_acquire_lock(); + void handle_pre_acquire_lock(int r); + void handle_acquire_lock(int r); + void handle_post_acquire_lock(int r); + void revert_to_unlock_state(int r); + + void send_reacquire_lock(); + void handle_reacquire_lock(int r); + + void send_release_lock(); + void handle_pre_release_lock(int r); + void handle_release_lock(int r); + void handle_post_release_lock(int r); + + void send_shutdown(); + void handle_shutdown(int r); + void send_shutdown_release(); + void handle_shutdown_pre_release(int r); + void handle_shutdown_post_release(int r); + void complete_shutdown(int r); +}; + +} // namespace librbd + +#endif // CEPH_LIBRBD_MANAGED_LOCK_H diff --git a/src/librbd/managed_lock/AcquireRequest.cc b/src/librbd/managed_lock/AcquireRequest.cc new file mode 100644 index 00000000000..591f8201be1 --- /dev/null +++ b/src/librbd/managed_lock/AcquireRequest.cc @@ -0,0 +1,317 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/managed_lock/AcquireRequest.h" +#include "librbd/Watcher.h" +#include "librbd/ManagedLock.h" +#include "cls/lock/cls_lock_client.h" +#include "cls/lock/cls_lock_types.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "include/stringify.h" +#include "librbd/Utils.h" + +#include "librbd/ImageCtx.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::managed_lock::AcquireRequest: " + +using std::string; + +namespace librbd { + +using util::detail::C_AsyncCallback; +using util::create_context_callback; +using util::create_rados_safe_callback; +using util::create_rados_ack_callback; + +namespace managed_lock { + +namespace { + +struct C_BlacklistClient : public Context { + librados::IoCtx& ioctx; + std::string locker_address; + Context *on_finish; + + C_BlacklistClient(librados::IoCtx& ioctx, const std::string &locker_address, + Context *on_finish) + : ioctx(ioctx), locker_address(locker_address), + on_finish(on_finish) { + } + + virtual void finish(int r) override { + librados::Rados rados(ioctx); + CephContext *cct = reinterpret_cast(ioctx.cct()); + r = rados.blacklist_add(locker_address, + cct->_conf->rbd_blacklist_expire_seconds); + on_finish->complete(r); + } +}; + +} // anonymous namespace + +template +AcquireRequest* AcquireRequest::create(librados::IoCtx& ioctx, + Watcher *watcher, + ContextWQ *work_queue, + const string& oid, + const string& cookie, + Context *on_finish) { + return new AcquireRequest(ioctx, watcher, work_queue, oid, cookie, + on_finish); +} + +template +AcquireRequest::AcquireRequest(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, const string& oid, + const string& cookie, Context *on_finish) + : m_ioctx(ioctx), m_watcher(watcher), + m_cct(reinterpret_cast(m_ioctx.cct())), + m_work_queue(work_queue), m_oid(oid), m_cookie(cookie), + m_on_finish(new C_AsyncCallback(work_queue, on_finish)), + m_error_result(0) { +} + +template +AcquireRequest::~AcquireRequest() { +} + +template +void AcquireRequest::send() { + send_lock(); +} + +template +void AcquireRequest::send_lock() { + ldout(m_cct, 10) << __func__ << dendl; + + librados::ObjectWriteOperation op; + rados::cls::lock::lock(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, m_cookie, + ManagedLock::WATCHER_LOCK_TAG, "", utime_t(), 0); + + using klass = AcquireRequest; + librados::AioCompletion *rados_completion = + create_rados_safe_callback(this); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +template +void AcquireRequest::handle_lock(int r) { + ldout(m_cct, 10) << __func__ << ": r=" << r << dendl; + + if (r == 0) { + finish(); + return; + } else if (r != -EBUSY) { + save_result(r); + lderr(m_cct) << "failed to lock: " << cpp_strerror(r) << dendl; + finish(); + return; + } + + send_get_lockers(); +} + +template +void AcquireRequest::send_get_lockers() { + ldout(m_cct, 10) << __func__ << dendl; + + librados::ObjectReadOperation op; + rados::cls::lock::get_lock_info_start(&op, RBD_LOCK_NAME); + + using klass = AcquireRequest; + librados::AioCompletion *rados_completion = + create_rados_ack_callback(this); + m_out_bl.clear(); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op, &m_out_bl); + assert(r == 0); + rados_completion->release(); +} + +template +void AcquireRequest::handle_get_lockers(int r) { + ldout(m_cct, 10) << __func__ << ": r=" << r << dendl; + + std::map lockers; + ClsLockType lock_type; + std::string lock_tag; + + if (r == 0) { + bufferlist::iterator it = m_out_bl.begin(); + r = rados::cls::lock::get_lock_info_finish(&it, &lockers, + &lock_type, &lock_tag); + } + + save_result(r); + if (r < 0) { + lderr(m_cct) << "failed to retrieve lockers: " << cpp_strerror(r) << dendl; + finish(); + return; + } + + if (lockers.empty()) { + ldout(m_cct, 20) << "no lockers detected" << dendl; + send_lock(); + return; + } + + if (lock_tag != ManagedLock::WATCHER_LOCK_TAG) { + ldout(m_cct, 5) <<"locked by external mechanism: tag=" << lock_tag << dendl; + save_result(-EBUSY); + finish(); + return; + } + + if (lock_type == LOCK_SHARED) { + ldout(m_cct, 5) << "shared lock type detected" << dendl; + save_result(-EBUSY); + finish(); + return; + } + + std::map::iterator iter = lockers.begin(); + if (!ManagedLock::decode_lock_cookie(iter->first.cookie, &m_locker_handle)) { + ldout(m_cct, 5) << "locked by external mechanism: " + << "cookie=" << iter->first.cookie << dendl; + save_result(-EBUSY); + finish(); + return; + } + + m_locker_entity = iter->first.locker; + m_locker_cookie = iter->first.cookie; + m_locker_address = stringify(iter->second.addr); + if (m_locker_cookie.empty() || m_locker_address.empty()) { + ldout(m_cct, 20) << "no valid lockers detected" << dendl; + send_lock(); + return; + } + + ldout(m_cct, 10) << "retrieved exclusive locker: " + << m_locker_entity << "@" << m_locker_address << dendl; + send_get_watchers(); +} + +template +void AcquireRequest::send_get_watchers() { + ldout(m_cct, 10) << __func__ << dendl; + + librados::ObjectReadOperation op; + op.list_watchers(&m_watchers, &m_watchers_ret_val); + + using klass = AcquireRequest; + librados::AioCompletion *rados_completion = + create_rados_ack_callback(this); + m_out_bl.clear(); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op, &m_out_bl); + assert(r == 0); + rados_completion->release(); +} + +template +void AcquireRequest::handle_get_watchers(int r) { + ldout(m_cct, 10) << __func__ << ": r=" << r << dendl; + + if (r == 0) { + r = m_watchers_ret_val; + } + save_result(r); + if (r < 0) { + lderr(m_cct) << "failed to retrieve watchers: " << cpp_strerror(r) << dendl; + finish(); + return; + } + + for (auto &watcher : m_watchers) { + if ((strncmp(m_locker_address.c_str(), + watcher.addr, sizeof(watcher.addr)) == 0) && + (m_locker_handle == watcher.cookie)) { + ldout(m_cct, 10) << "lock owner is still alive" << dendl; + + save_result(-EAGAIN); + finish(); + return; + } + } + + send_blacklist(); +} + +template +void AcquireRequest::send_blacklist() { + if (!m_cct->_conf->rbd_blacklist_on_break_lock) { + send_break_lock(); + return; + } + ldout(m_cct, 10) << __func__ << dendl; + + // TODO: need async version of RadosClient::blacklist_add + using klass = AcquireRequest; + Context *ctx = create_context_callback( + this); + m_work_queue->queue( + new C_BlacklistClient(m_ioctx, m_locker_address, ctx), 0); +} +template +void AcquireRequest::handle_blacklist(int r) { + ldout(m_cct, 10) << __func__ << ": r=" << r << dendl; + + save_result(r); + if (r < 0) { + lderr(m_cct) << "failed to blacklist lock owner: " << cpp_strerror(r) + << dendl; + finish(); + return; + } + send_break_lock(); +} + +template +void AcquireRequest::send_break_lock() { + ldout(m_cct, 10) << __func__ << dendl; + + librados::ObjectWriteOperation op; + rados::cls::lock::break_lock(&op, RBD_LOCK_NAME, m_locker_cookie, + m_locker_entity); + + using klass = AcquireRequest; + librados::AioCompletion *rados_completion = + create_rados_safe_callback(this); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +template +void AcquireRequest::handle_break_lock(int r) { + ldout(m_cct, 10) << __func__ << ": r=" << r << dendl; + + if (r == -ENOENT) { + r = 0; + } else if (r < 0) { + lderr(m_cct) << "failed to break lock: " << cpp_strerror(r) << dendl; + save_result(r); + finish(); + return; + } + + send_lock(); +} + +template +void AcquireRequest::finish() { + m_on_finish->complete(m_error_result); + delete this; +} + +} // namespace managed_lock +} // namespace librbd + +template class librbd::managed_lock::AcquireRequest; diff --git a/src/librbd/managed_lock/AcquireRequest.h b/src/librbd/managed_lock/AcquireRequest.h new file mode 100644 index 00000000000..c8a8a626d36 --- /dev/null +++ b/src/librbd/managed_lock/AcquireRequest.h @@ -0,0 +1,124 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MANAGED_LOCK_ACQUIRE_REQUEST_H +#define CEPH_LIBRBD_MANAGED_LOCK_ACQUIRE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "include/int_types.h" +#include "include/buffer.h" +#include "msg/msg_types.h" +#include "librbd/watcher/Types.h" +#include + +class Context; +class ContextWQ; + +namespace librbd { + +class Watcher; + +namespace managed_lock { + +template +class AcquireRequest { +private: + typedef watcher::Traits TypeTraits; + typedef typename TypeTraits::Watcher Watcher; + +public: + static AcquireRequest* create(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, const std::string& oid, + const std::string& cookie, Context *on_finish); + + ~AcquireRequest(); + void send(); + +private: + + /** + * @verbatim + * + * + * | + * | + * | + * | /-----------------------------------------------------------\ + * | | | + * | | (no lockers) | + * | | . . . . . . . . . . . . . . . . . . . . . . | + * | | . . | + * | v v (EBUSY) . | + * \--> LOCK_IMAGE * * * * * * * * > GET_LOCKERS . . . . | + * | | | + * | v | + * | GET_WATCHERS | + * | | | + * | v | + * | BLACKLIST (skip if blacklist | + * | | disabled) | + * | v | + * | BREAK_LOCK | + * | | | + * | \-----------------------------/ + * v + * + * + * @endverbatim + */ + + AcquireRequest(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, const std::string& oid, + const std::string& cookie, Context *on_finish); + + librados::IoCtx& m_ioctx; + Watcher *m_watcher; + CephContext *m_cct; + ContextWQ *m_work_queue; + std::string m_oid; + std::string m_cookie; + Context *m_on_finish; + + bufferlist m_out_bl; + + std::list m_watchers; + int m_watchers_ret_val; + + entity_name_t m_locker_entity; + std::string m_locker_cookie; + std::string m_locker_address; + uint64_t m_locker_handle; + + int m_error_result; + + void send_lock(); + void handle_lock(int r); + + void send_unlock(); + void handle_unlock(int r); + + void send_get_lockers(); + void handle_get_lockers(int r); + + void send_get_watchers(); + void handle_get_watchers(int r); + + void send_blacklist(); + void handle_blacklist(int r); + + void send_break_lock(); + void handle_break_lock(int r); + + void finish(); + + void save_result(int r) { + if (m_error_result == 0 && r < 0) { + m_error_result = r; + } + } +}; + +} // namespace managed_lock +} // namespace librbd + +#endif // CEPH_LIBRBD_MANAGED_LOCK_ACQUIRE_REQUEST_H diff --git a/src/librbd/managed_lock/ReacquireRequest.cc b/src/librbd/managed_lock/ReacquireRequest.cc new file mode 100644 index 00000000000..33bd1db7136 --- /dev/null +++ b/src/librbd/managed_lock/ReacquireRequest.cc @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/managed_lock/ReacquireRequest.h" +#include "librbd/Watcher.h" +#include "librbd/ManagedLock.h" +#include "cls/lock/cls_lock_client.h" +#include "cls/lock/cls_lock_types.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/Utils.h" + +#include "librbd/ImageCtx.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::managed_lock::ReacquireRequest: " \ + << this << ": " << __func__ + +using std::string; + +namespace librbd { +namespace managed_lock { + +using librbd::util::create_rados_safe_callback; + +template +ReacquireRequest::ReacquireRequest(librados::IoCtx& ioctx, + const string& oid, + const string& old_cookie, + const string &new_cookie, + Context *on_finish) + : m_ioctx(ioctx), m_oid(oid), m_old_cookie(old_cookie), + m_new_cookie(new_cookie), m_on_finish(on_finish) { +} + + +template +void ReacquireRequest::send() { + set_cookie(); +} + +template +void ReacquireRequest::set_cookie() { + CephContext *cct = reinterpret_cast(m_ioctx.cct()); + ldout(cct, 10) << dendl; + + librados::ObjectWriteOperation op; + rados::cls::lock::set_cookie(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, m_old_cookie, + ManagedLock::WATCHER_LOCK_TAG, m_new_cookie); + + librados::AioCompletion *rados_completion = create_rados_safe_callback< + ReacquireRequest, &ReacquireRequest::handle_set_cookie>(this); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +template +void ReacquireRequest::handle_set_cookie(int r) { + CephContext *cct = reinterpret_cast(m_ioctx.cct()); + ldout(cct, 10) << ": r=" << r << dendl; + + if (r == -EOPNOTSUPP) { + ldout(cct, 10) << ": OSD doesn't support updating lock" << dendl; + } else if (r < 0) { + lderr(cct) << ": failed to update lock: " << cpp_strerror(r) << dendl; + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace managed_lock +} // namespace librbd + +template class librbd::managed_lock::ReacquireRequest; diff --git a/src/librbd/managed_lock/ReacquireRequest.h b/src/librbd/managed_lock/ReacquireRequest.h new file mode 100644 index 00000000000..60256c836de --- /dev/null +++ b/src/librbd/managed_lock/ReacquireRequest.h @@ -0,0 +1,65 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MANAGED_LOCK_REACQUIRE_REQUEST_H +#define CEPH_LIBRBD_MANAGED_LOCK_REACQUIRE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "include/int_types.h" +#include + +class Context; + +namespace librbd { + +class Watcher; + +namespace managed_lock { + +template +class ReacquireRequest { +public: + + static ReacquireRequest *create(librados::IoCtx& ioctx, + const std::string& oid, + const std::string& old_cookie, + const std::string &new_cookie, + Context *on_finish) { + return new ReacquireRequest(ioctx, oid, old_cookie, new_cookie, on_finish); + } + + ReacquireRequest(librados::IoCtx& ioctx, const std::string& oid, + const std::string& old_cookie, + const std::string &new_cookie, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * SET_COOKIE + * | + * v + * + * + * @endverbatim + */ + librados::IoCtx& m_ioctx; + std::string m_oid; + std::string m_old_cookie; + std::string m_new_cookie; + Context *m_on_finish; + + void set_cookie(); + void handle_set_cookie(int r); + +}; + +} // namespace managed_lock +} // namespace librbd + +#endif // CEPH_LIBRBD_MANAGED_LOCK_REACQUIRE_REQUEST_H diff --git a/src/librbd/managed_lock/ReleaseRequest.cc b/src/librbd/managed_lock/ReleaseRequest.cc new file mode 100644 index 00000000000..84446a77041 --- /dev/null +++ b/src/librbd/managed_lock/ReleaseRequest.cc @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/managed_lock/ReleaseRequest.h" +#include "librbd/Watcher.h" +#include "cls/lock/cls_lock_client.h" +#include "cls/lock/cls_lock_types.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/Utils.h" + +#include "librbd/ImageCtx.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::managed_lock::ReleaseRequest: " + +namespace librbd { +namespace managed_lock { + +using util::detail::C_AsyncCallback; +using util::create_context_callback; +using util::create_rados_safe_callback; + +template +ReleaseRequest* ReleaseRequest::create(librados::IoCtx& ioctx, + Watcher *watcher, + ContextWQ *work_queue, + const string& oid, + const string& cookie, + Context *on_finish) { + return new ReleaseRequest(ioctx, watcher, work_queue, oid, cookie, + on_finish); +} + +template +ReleaseRequest::ReleaseRequest(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, const string& oid, + const string& cookie, Context *on_finish) + : m_ioctx(ioctx), m_watcher(watcher), m_oid(oid), m_cookie(cookie), + m_on_finish(new C_AsyncCallback(work_queue, on_finish)) { +} + +template +ReleaseRequest::~ReleaseRequest() { +} + + +template +void ReleaseRequest::send() { + send_unlock(); +} + +template +void ReleaseRequest::send_unlock() { + CephContext *cct = reinterpret_cast(m_ioctx.cct()); + ldout(cct, 10) << __func__ << dendl; + + librados::ObjectWriteOperation op; + rados::cls::lock::unlock(&op, RBD_LOCK_NAME, m_cookie); + + using klass = ReleaseRequest; + librados::AioCompletion *rados_completion = + create_rados_safe_callback(this); + int r = m_ioctx.aio_operate(m_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +template +void ReleaseRequest::handle_unlock(int r) { + CephContext *cct = reinterpret_cast(m_ioctx.cct()); + ldout(cct, 10) << __func__ << ": r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to unlock: " << cpp_strerror(r) << dendl; + } + + finish(); +} + +template +void ReleaseRequest::finish() { + m_on_finish->complete(0); + delete this; +} + +} // namespace managed_lock +} // namespace librbd + +template class librbd::managed_lock::ReleaseRequest; + diff --git a/src/librbd/managed_lock/ReleaseRequest.h b/src/librbd/managed_lock/ReleaseRequest.h new file mode 100644 index 00000000000..89205136ecf --- /dev/null +++ b/src/librbd/managed_lock/ReleaseRequest.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MANAGED_LOCK_RELEASE_REQUEST_H +#define CEPH_LIBRBD_MANAGED_LOCK_RELEASE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "librbd/watcher/Types.h" +#include + +class Context; +class ContextWQ; + +namespace librbd { + +class Watcher; + +namespace managed_lock { + +template +class ReleaseRequest { +private: + typedef watcher::Traits TypeTraits; + typedef typename TypeTraits::Watcher Watcher; + +public: + static ReleaseRequest* create(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, + const std::string& oid, + const std::string& cookie, + Context *on_finish); + + ~ReleaseRequest(); + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * UNLOCK + * | + * v + * + * + * @endverbatim + */ + + ReleaseRequest(librados::IoCtx& ioctx, Watcher *watcher, + ContextWQ *work_queue, const std::string& oid, + const std::string& cookie, Context *on_finish); + + librados::IoCtx& m_ioctx; + Watcher *m_watcher; + std::string m_oid; + std::string m_cookie; + Context *m_on_finish; + + void send_unlock(); + void handle_unlock(int r); + + void finish(); + +}; + +} // namespace managed_lock +} // namespace librbd + +#endif // CEPH_LIBRBD_MANAGED_LOCK_RELEASE_REQUEST_H diff --git a/src/librbd/watcher/Types.h b/src/librbd/watcher/Types.h index d9823ca31d4..0f11ab8885a 100644 --- a/src/librbd/watcher/Types.h +++ b/src/librbd/watcher/Types.h @@ -79,6 +79,10 @@ private: bufferlist::iterator &m_iter; }; +template +struct Traits { + typedef librbd::Watcher Watcher; +}; } // namespace watcher } // namespace librbd