From d1541d6ed2365e78110c176e69ef5132651efa6f Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 12 Aug 2015 13:36:42 -0400 Subject: [PATCH] librbd: migrate object map snapshot ops to async state machines Since all snapshot handling was migrated to async state machines, the object map snapshot handling also needs to be converted. Signed-off-by: Jason Dillaman --- src/librbd/Makefile.am | 6 + src/librbd/ObjectMap.cc | 187 +++-------------- src/librbd/ObjectMap.h | 8 +- src/librbd/object_map/InvalidateRequest.cc | 1 + src/librbd/object_map/Request.cc | 5 +- .../object_map/SnapshotCreateRequest.cc | 149 ++++++++++++++ src/librbd/object_map/SnapshotCreateRequest.h | 79 +++++++ .../object_map/SnapshotRemoveRequest.cc | 193 ++++++++++++++++++ src/librbd/object_map/SnapshotRemoveRequest.h | 89 ++++++++ .../object_map/SnapshotRollbackRequest.cc | 129 ++++++++++++ .../object_map/SnapshotRollbackRequest.h | 77 +++++++ src/librbd/operation/SnapshotCreateRequest.cc | 38 +++- src/librbd/operation/SnapshotCreateRequest.h | 4 + src/librbd/operation/SnapshotRemoveRequest.cc | 15 +- .../operation/SnapshotRollbackRequest.cc | 15 +- 15 files changed, 822 insertions(+), 173 deletions(-) create mode 100644 src/librbd/object_map/SnapshotCreateRequest.cc create mode 100644 src/librbd/object_map/SnapshotCreateRequest.h create mode 100644 src/librbd/object_map/SnapshotRemoveRequest.cc create mode 100644 src/librbd/object_map/SnapshotRemoveRequest.h create mode 100644 src/librbd/object_map/SnapshotRollbackRequest.cc create mode 100644 src/librbd/object_map/SnapshotRollbackRequest.h diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am index fae8664bab25b..ee3b1e829d57a 100644 --- a/src/librbd/Makefile.am +++ b/src/librbd/Makefile.am @@ -28,6 +28,9 @@ librbd_internal_la_SOURCES = \ librbd/object_map/InvalidateRequest.cc \ librbd/object_map/Request.cc \ librbd/object_map/ResizeRequest.cc \ + librbd/object_map/SnapshotCreateRequest.cc \ + librbd/object_map/SnapshotRemoveRequest.cc \ + librbd/object_map/SnapshotRollbackRequest.cc \ librbd/object_map/UpdateRequest.cc \ librbd/operation/FlattenRequest.cc \ librbd/operation/RebuildObjectMapRequest.cc \ @@ -90,6 +93,9 @@ noinst_HEADERS += \ librbd/object_map/InvalidateRequest.h \ librbd/object_map/Request.h \ librbd/object_map/ResizeRequest.h \ + librbd/object_map/SnapshotCreateRequest.h \ + librbd/object_map/SnapshotRemoveRequest.h \ + librbd/object_map/SnapshotRollbackRequest.h \ librbd/object_map/UpdateRequest.h \ librbd/operation/FlattenRequest.h \ librbd/operation/RebuildObjectMapRequest.h \ diff --git a/src/librbd/ObjectMap.cc b/src/librbd/ObjectMap.cc index d070488b47141..6c9ebc7317088 100644 --- a/src/librbd/ObjectMap.cc +++ b/src/librbd/ObjectMap.cc @@ -6,6 +6,9 @@ #include "librbd/internal.h" #include "librbd/object_map/InvalidateRequest.h" #include "librbd/object_map/ResizeRequest.h" +#include "librbd/object_map/SnapshotCreateRequest.h" +#include "librbd/object_map/SnapshotRemoveRequest.h" +#include "librbd/object_map/SnapshotRollbackRequest.h" #include "librbd/object_map/UpdateRequest.h" #include "common/dout.h" #include "common/errno.h" @@ -60,6 +63,11 @@ bool ObjectMap::enabled() const return m_enabled; } +bool ObjectMap::enabled(const RWLock &object_map_lock) const { + assert(m_image_ctx.object_map_lock.is_locked()); + return m_enabled; +} + int ObjectMap::lock() { if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)) { @@ -243,174 +251,35 @@ void ObjectMap::refresh(uint64_t snap_id) } } -void ObjectMap::rollback(uint64_t snap_id) { - assert(m_image_ctx.snap_lock.is_wlocked()); - int r; - std::string oid(object_map_name(m_image_ctx.id, CEPH_NOSNAP)); - - CephContext *cct = m_image_ctx.cct; - ldout(cct, 10) << &m_image_ctx << " rollback object map" << dendl; - - if ((m_image_ctx.features & RBD_FEATURE_OBJECT_MAP) == 0) { - r = m_image_ctx.md_ctx.remove(oid); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "unable to remove object map: " << cpp_strerror(r) - << dendl; - } - return; - } - - RWLock::WLocker l(m_image_ctx.object_map_lock); - if (!m_enabled) { - return; - } - - std::string snap_oid(object_map_name(m_image_ctx.id, snap_id)); - bufferlist bl; - r = m_image_ctx.md_ctx.read(snap_oid, bl, 0, 0); - if (r < 0) { - lderr(cct) << "unable to load snapshot object map '" << snap_oid << "': " - << cpp_strerror(r) << dendl; - invalidate(snap_id, false); - return; - } - - librados::ObjectWriteOperation op; - rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", ""); - op.write_full(bl); +void ObjectMap::rollback(uint64_t snap_id, Context *on_finish) { + assert(m_image_ctx.snap_lock.is_locked()); + assert(m_image_ctx.object_map_lock.is_wlocked()); + assert(m_enabled); - r = m_image_ctx.md_ctx.operate(oid, &op); - if (r < 0) { - lderr(cct) << "unable to rollback object map: " << cpp_strerror(r) - << dendl; - invalidate(CEPH_NOSNAP, true); - } + object_map::SnapshotRollbackRequest *req = + new object_map::SnapshotRollbackRequest(m_image_ctx, snap_id, on_finish); + req->send(); } -void ObjectMap::snapshot_add(uint64_t snap_id) { - assert(m_image_ctx.snap_lock.is_wlocked()); - if ((m_image_ctx.features & RBD_FEATURE_OBJECT_MAP) == 0) { - return; - } - - CephContext *cct = m_image_ctx.cct; - ldout(cct, 10) << &m_image_ctx << " snapshot object map" << dendl; - - int r; - bufferlist bl; - RWLock::WLocker l(m_image_ctx.object_map_lock); - if (!m_enabled) { - return; - } - std::string oid(object_map_name(m_image_ctx.id, CEPH_NOSNAP)); - r = m_image_ctx.md_ctx.read(oid, bl, 0, 0); - if (r < 0) { - lderr(cct) << "unable to load object map: " << cpp_strerror(r) - << dendl; - invalidate(CEPH_NOSNAP, false); - return; - } - - std::string snap_oid(object_map_name(m_image_ctx.id, snap_id)); - r = m_image_ctx.md_ctx.write_full(snap_oid, bl); - if (r < 0) { - lderr(cct) << "unable to snapshot object map '" << snap_oid << "': " - << cpp_strerror(r) << dendl; - invalidate(snap_id, false); - return; - } - - if ((m_image_ctx.features & RBD_FEATURE_FAST_DIFF) != 0) { - librados::ObjectWriteOperation op; - rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", ""); - cls_client::object_map_snap_add(&op); - r = m_image_ctx.md_ctx.operate(oid, &op); - if (r < 0) { - lderr(cct) << "unable to snapshot object map: " << cpp_strerror(r) - << dendl; - invalidate(CEPH_NOSNAP, true); - return; - } +void ObjectMap::snapshot_add(uint64_t snap_id, Context *on_finish) { + assert(m_image_ctx.snap_lock.is_locked()); + assert((m_image_ctx.features & RBD_FEATURE_OBJECT_MAP) != 0); - for (uint64_t i = 0; i < m_object_map.size(); ++i) { - if (m_object_map[i] == OBJECT_EXISTS) { - m_object_map[i] = OBJECT_EXISTS_CLEAN; - } - } - } + object_map::SnapshotCreateRequest *req = + new object_map::SnapshotCreateRequest(m_image_ctx, &m_object_map, snap_id, + on_finish); + req->send(); } -int ObjectMap::snapshot_remove(uint64_t snap_id) { +void ObjectMap::snapshot_remove(uint64_t snap_id, Context *on_finish) { assert(m_image_ctx.snap_lock.is_wlocked()); + assert((m_image_ctx.features & RBD_FEATURE_OBJECT_MAP) != 0); assert(snap_id != CEPH_NOSNAP); - CephContext *cct = m_image_ctx.cct; - - int r; - if ((m_image_ctx.features & RBD_FEATURE_FAST_DIFF) != 0) { - RWLock::WLocker l(m_image_ctx.object_map_lock); - - uint64_t next_snap_id = CEPH_NOSNAP; - std::map::const_iterator it = - m_image_ctx.snap_info.find(snap_id); - assert(it != m_image_ctx.snap_info.end()); - - ++it; - if (it != m_image_ctx.snap_info.end()) { - next_snap_id = it->first; - } - ceph::BitVector<2> snap_object_map; - std::string snap_oid(object_map_name(m_image_ctx.id, snap_id)); - r = cls_client::object_map_load(&m_image_ctx.md_ctx, snap_oid, - &snap_object_map); - if (r < 0) { - lderr(cct) << "error loading snapshot object map: " << cpp_strerror(r) - << dendl; - } - - if (r == 0) { - uint64_t flags; - m_image_ctx.get_flags(snap_id, &flags); - if ((flags & RBD_FLAG_OBJECT_MAP_INVALID) != 0) { - invalidate(next_snap_id, true); - r = -EINVAL; - } - } - - if (r == 0) { - std::string oid(object_map_name(m_image_ctx.id, next_snap_id)); - librados::ObjectWriteOperation op; - if (next_snap_id == CEPH_NOSNAP) { - rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", - ""); - } - cls_client::object_map_snap_remove(&op, snap_object_map); - - r = m_image_ctx.md_ctx.operate(oid, &op); - if (r < 0) { - lderr(cct) << "unable to remove object map snapshot: " - << cpp_strerror(r) << dendl; - invalidate(next_snap_id, true); - } - } - - if (r == 0 && next_snap_id == CEPH_NOSNAP) { - for (uint64_t i = 0; i < m_object_map.size(); ++i) { - if (m_object_map[i] == OBJECT_EXISTS_CLEAN && - (i >= snap_object_map.size() || - snap_object_map[i] == OBJECT_EXISTS)) { - m_object_map[i] = OBJECT_EXISTS; - } - } - } - } - - std::string oid(object_map_name(m_image_ctx.id, snap_id)); - r = m_image_ctx.md_ctx.remove(oid); - if (r < 0 && r != -ENOENT) { - return r; - } - return 0; + object_map::SnapshotRemoveRequest *req = + new object_map::SnapshotRemoveRequest(m_image_ctx, &m_object_map, snap_id, + on_finish); + req->send(); } void ObjectMap::aio_save(Context *on_finish) diff --git a/src/librbd/ObjectMap.h b/src/librbd/ObjectMap.h index b175c3669ff2a..46c5b64cf50bb 100644 --- a/src/librbd/ObjectMap.h +++ b/src/librbd/ObjectMap.h @@ -10,6 +10,7 @@ #include class Context; +class RWLock; namespace librbd { @@ -52,11 +53,12 @@ public: Context *on_finish); void refresh(uint64_t snap_id); - void rollback(uint64_t snap_id); - void snapshot_add(uint64_t snap_id); - int snapshot_remove(uint64_t snap_id); + void rollback(uint64_t snap_id, Context *on_finish); + void snapshot_add(uint64_t snap_id, Context *on_finish); + void snapshot_remove(uint64_t snap_id, Context *on_finish); bool enabled() const; + bool enabled(const RWLock &object_map_lock) const; private: ImageCtx &m_image_ctx; diff --git a/src/librbd/object_map/InvalidateRequest.cc b/src/librbd/object_map/InvalidateRequest.cc index 63521280cb41c..464a5dacc86df 100644 --- a/src/librbd/object_map/InvalidateRequest.cc +++ b/src/librbd/object_map/InvalidateRequest.cc @@ -15,6 +15,7 @@ namespace librbd { namespace object_map { void InvalidateRequest::send() { + assert(m_image_ctx.owner_lock.is_locked()); assert(m_image_ctx.snap_lock.is_wlocked()); uint64_t snap_flags; diff --git a/src/librbd/object_map/Request.cc b/src/librbd/object_map/Request.cc index d78cb834a89d2..5b129b97f3011 100644 --- a/src/librbd/object_map/Request.cc +++ b/src/librbd/object_map/Request.cc @@ -25,10 +25,7 @@ bool Request::should_complete(int r) { switch (m_state) { case STATE_REQUEST: - if (r == -EBUSY) { - lderr(cct) << "object map lock not owned by client" << dendl; - return invalidate(); - } else if (r < 0) { + if (r < 0) { lderr(cct) << "failed to update object map: " << cpp_strerror(r) << dendl; return invalidate(); diff --git a/src/librbd/object_map/SnapshotCreateRequest.cc b/src/librbd/object_map/SnapshotCreateRequest.cc new file mode 100644 index 0000000000000..9bf4a7913c65e --- /dev/null +++ b/src/librbd/object_map/SnapshotCreateRequest.cc @@ -0,0 +1,149 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/object_map/SnapshotCreateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" +#include "cls/lock/cls_lock_client.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::object_map::SnapshotCreateRequest: " + +namespace librbd { +namespace object_map { + +namespace { + +std::ostream& operator<<(std::ostream& os, + const SnapshotCreateRequest::State& state) { + switch(state) { + case SnapshotCreateRequest::STATE_READ_MAP: + os << "READ_MAP"; + break; + case SnapshotCreateRequest::STATE_WRITE_MAP: + os << "WRITE_MAP"; + break; + case SnapshotCreateRequest::STATE_ADD_SNAPSHOT: + os << "ADD_SNAPSHOT"; + break; + default: + os << "UNKNOWN (" << static_cast(state) << ")"; + break; + } + return os; +} + +} // anonymous namespace + +void SnapshotCreateRequest::send() { + send_read_map(); +} + +bool SnapshotCreateRequest::should_complete(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << ": state=" << m_state << ", " + << "r=" << r << dendl; + if (r < 0 && m_ret_val == 0) { + m_ret_val = r; + } + if (m_ret_val < 0) { + // pass errors down to base class to invalidate the object map + return Request::should_complete(r); + } + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + bool finished = false; + switch (m_state) { + case STATE_READ_MAP: + send_write_map(); + break; + case STATE_WRITE_MAP: + finished = send_add_snapshot(); + break; + case STATE_ADD_SNAPSHOT: + update_object_map(); + finished = true; + break; + default: + assert(false); + break; + } + return finished; +} + +void SnapshotCreateRequest::send_read_map() { + assert(m_image_ctx.snap_lock.is_locked()); + assert(m_image_ctx.get_snap_info(m_snap_id) != NULL); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_READ_MAP; + + // IO is blocked due to the snapshot creation -- consistent to read from disk + librados::ObjectReadOperation op; + op.read(0, 0, NULL, NULL); + + std::string oid(ObjectMap::object_map_name(m_image_ctx.id, CEPH_NOSNAP)); + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(oid, rados_completion, &op, + &m_read_bl); + assert(r == 0); + rados_completion->release(); +} + +void SnapshotCreateRequest::send_write_map() { + std::string snap_oid(ObjectMap::object_map_name(m_image_ctx.id, m_snap_id)); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << ": snap_oid=" << snap_oid + << dendl; + m_state = STATE_WRITE_MAP; + + librados::ObjectWriteOperation op; + op.write_full(m_read_bl); + + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(snap_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +bool SnapshotCreateRequest::send_add_snapshot() { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + if ((m_image_ctx.features & RBD_FEATURE_FAST_DIFF) == 0) { + return true; + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_ADD_SNAPSHOT; + + librados::ObjectWriteOperation op; + rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", ""); + cls_client::object_map_snap_add(&op); + + std::string oid(ObjectMap::object_map_name(m_image_ctx.id, CEPH_NOSNAP)); + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); + return false; +} + +void SnapshotCreateRequest::update_object_map() { + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + RWLock::WLocker object_map_locker(m_image_ctx.object_map_lock); + + for (uint64_t i = 0; i < m_object_map.size(); ++i) { + if (m_object_map[i] == OBJECT_EXISTS) { + m_object_map[i] = OBJECT_EXISTS_CLEAN; + } + } +} + +} // namespace object_map +} // namespace librbd diff --git a/src/librbd/object_map/SnapshotCreateRequest.h b/src/librbd/object_map/SnapshotCreateRequest.h new file mode 100644 index 0000000000000..f8143325d548a --- /dev/null +++ b/src/librbd/object_map/SnapshotCreateRequest.h @@ -0,0 +1,79 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_CREATE_REQUEST_H +#define CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_CREATE_REQUEST_H + +#include "include/int_types.h" +#include "common/bit_vector.hpp" +#include "librbd/object_map/Request.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace object_map { + +class SnapshotCreateRequest : public Request { +public: + /** + * Snapshot create goes through the following state machine: + * + * @verbatim + * + * + * | + * v + * STATE_READ_MAP + * | + * v (skip) + * STATE_WRITE_MAP . . . . . . . + * | . + * v v + * STATE_ADD_SNAPSHOT ---> + * + * @endverbatim + * + * The _ADD_SNAPSHOT state is skipped if the FAST_DIFF feature isn't enabled. + */ + enum State { + STATE_READ_MAP, + STATE_WRITE_MAP, + STATE_ADD_SNAPSHOT + }; + + SnapshotCreateRequest(ImageCtx &image_ctx, ceph::BitVector<2> *object_map, + uint64_t snap_id, Context *on_finish) + : Request(image_ctx, snap_id, on_finish), + m_object_map(*object_map), m_ret_val(0) { + } + + virtual void send(); + +protected: + virtual bool should_complete(int r); + + virtual void finish() { + } + +private: + State m_state; + ceph::BitVector<2> &m_object_map; + + bufferlist m_read_bl; + int m_ret_val; + + void send_read_map(); + void send_write_map(); + bool send_add_snapshot(); + + void update_object_map(); + +}; + +} // namespace object_map +} // namespace librbd + +#endif // CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_CREATE_REQUEST_H diff --git a/src/librbd/object_map/SnapshotRemoveRequest.cc b/src/librbd/object_map/SnapshotRemoveRequest.cc new file mode 100644 index 0000000000000..679735f0f34ed --- /dev/null +++ b/src/librbd/object_map/SnapshotRemoveRequest.cc @@ -0,0 +1,193 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/object_map/SnapshotRemoveRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" +#include "librbd/object_map/InvalidateRequest.h" +#include "cls/lock/cls_lock_client.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::object_map::SnapshotRemoveRequest: " + +namespace librbd { +namespace object_map { + +namespace { + +std::ostream& operator<<(std::ostream& os, + const SnapshotRemoveRequest::State& state) { + switch(state) { + case SnapshotRemoveRequest::STATE_LOAD_MAP: + os << "LOAD_MAP"; + break; + case SnapshotRemoveRequest::STATE_REMOVE_SNAPSHOT: + os << "REMOVE_SNAPSHOT"; + break; + case SnapshotRemoveRequest::STATE_INVALIDATE_NEXT_MAP: + os << "INVALIDATE_NEXT_MAP"; + break; + case SnapshotRemoveRequest::STATE_REMOVE_MAP: + os << "REMOVE_MAP"; + break; + default: + os << "UNKNOWN (" << static_cast(state) << ")"; + break; + } + return os; +} + +} // anonymous namespace + +void SnapshotRemoveRequest::send() { + assert(m_image_ctx.owner_lock.is_locked()); + assert(m_image_ctx.snap_lock.is_wlocked()); + + if ((m_image_ctx.features & RBD_FEATURE_FAST_DIFF) != 0) { + compute_next_snap_id(); + + uint64_t flags; + int r = m_image_ctx.get_flags(m_snap_id, &flags); + assert(r == 0); + + if ((flags & RBD_FLAG_OBJECT_MAP_INVALID) != 0) { + send_invalidate_next_map(); + } else { + send_load_map(); + } + } else { + send_remove_map(); + } +} + +bool SnapshotRemoveRequest::should_complete(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << ": state=" << m_state << ", " + << "r=" << r << dendl; + + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + bool finished = false; + switch (m_state) { + case STATE_LOAD_MAP: + if (r < 0) { + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + send_invalidate_next_map(); + } else { + send_remove_snapshot(); + } + break; + case STATE_REMOVE_SNAPSHOT: + if (r < 0 && r != -ENOENT) { + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + send_invalidate_next_map(); + } else { + update_object_map(); + send_remove_map(); + } + break; + case STATE_INVALIDATE_NEXT_MAP: + send_remove_map(); + break; + case STATE_REMOVE_MAP: + finished = true; + break; + default: + assert(false); + break; + } + return finished; +} + +void SnapshotRemoveRequest::send_load_map() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_LOAD_MAP; + + std::string snap_oid(ObjectMap::object_map_name(m_image_ctx.id, m_snap_id)); + cls_client::object_map_load(&m_image_ctx.md_ctx, snap_oid, + &m_snap_object_map, create_callback_context()); +} + +void SnapshotRemoveRequest::send_remove_snapshot() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_REMOVE_SNAPSHOT; + + librados::ObjectWriteOperation op; + if (m_next_snap_id == CEPH_NOSNAP) { + rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", ""); + } + cls_client::object_map_snap_remove(&op, m_snap_object_map); + + std::string oid(ObjectMap::object_map_name(m_image_ctx.id, m_next_snap_id)); + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +void SnapshotRemoveRequest::send_invalidate_next_map() { + assert(m_image_ctx.owner_lock.is_locked()); + assert(m_image_ctx.snap_lock.is_wlocked()); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_INVALIDATE_NEXT_MAP; + + InvalidateRequest *req = new InvalidateRequest(m_image_ctx, m_next_snap_id, + true, + create_callback_context()); + req->send(); +} + +void SnapshotRemoveRequest::send_remove_map() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_REMOVE_MAP; + + librados::ObjectWriteOperation op; + op.remove(); + + std::string oid(ObjectMap::object_map_name(m_image_ctx.id, m_snap_id)); + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +void SnapshotRemoveRequest::compute_next_snap_id() { + assert(m_image_ctx.snap_lock.is_locked()); + + m_next_snap_id = CEPH_NOSNAP; + std::map::const_iterator it = + m_image_ctx.snap_info.find(m_snap_id); + assert(it != m_image_ctx.snap_info.end()); + + ++it; + if (it != m_image_ctx.snap_info.end()) { + m_next_snap_id = it->first; + } +} + +void SnapshotRemoveRequest::update_object_map() { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::WLocker object_map_locker(m_image_ctx.object_map_lock); + if (m_next_snap_id == m_image_ctx.snap_id && m_next_snap_id == CEPH_NOSNAP) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + + for (uint64_t i = 0; i < m_object_map.size(); ++i) { + if (m_object_map[i] == OBJECT_EXISTS_CLEAN && + (i >= m_snap_object_map.size() || + m_snap_object_map[i] == OBJECT_EXISTS)) { + m_object_map[i] = OBJECT_EXISTS; + } + } + } +} + +} // namespace object_map +} // namespace librbd diff --git a/src/librbd/object_map/SnapshotRemoveRequest.h b/src/librbd/object_map/SnapshotRemoveRequest.h new file mode 100644 index 0000000000000..24950faef9e65 --- /dev/null +++ b/src/librbd/object_map/SnapshotRemoveRequest.h @@ -0,0 +1,89 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_REMOVE_REQUEST_H +#define CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_REMOVE_REQUEST_H + +#include "include/int_types.h" +#include "common/bit_vector.hpp" +#include "librbd/AsyncRequest.h" + +namespace librbd { +namespace object_map { + +class SnapshotRemoveRequest : public AsyncRequest<> { +public: + /** + * Snapshot rollback goes through the following state machine: + * + * @verbatim + * + * -----------> STATE_LOAD_MAP ----\ + * . * | + * . * (error) | + * . (invalid object map) v | + * . . . > STATE_INVALIDATE_NEXT_MAP | + * . | | + * . | | + * . (fast diff disabled) v v + * . . . . . . . . . . > STATE_REMOVE_MAP + * | + * v + * + * + * @endverbatim + * + * The _LOAD_MAP state is skipped if the fast diff feature is disabled. + * If the fast diff feature is enabled and the snapshot is flagged as + * invalid, the next snapshot / HEAD object mapis flagged as invalid; + * otherwise, the state machine proceeds to remove the object map. + */ + enum State { + STATE_LOAD_MAP, + STATE_REMOVE_SNAPSHOT, + STATE_INVALIDATE_NEXT_MAP, + STATE_REMOVE_MAP + }; + + SnapshotRemoveRequest(ImageCtx &image_ctx, ceph::BitVector<2> *object_map, + uint64_t snap_id, Context *on_finish) + : AsyncRequest(image_ctx, on_finish), m_object_map(*object_map), + m_snap_id(snap_id), m_next_snap_id(CEPH_NOSNAP) { + } + + virtual void send(); + +protected: + virtual bool should_complete(int r); + + virtual int filter_return_code(int r) const { + if (m_state == STATE_REMOVE_MAP && r == -ENOENT) { + return 0; + } + return r; + } + + virtual void finish() { + } + +private: + State m_state; + ceph::BitVector<2> &m_object_map; + uint64_t m_snap_id; + uint64_t m_next_snap_id; + + ceph::BitVector<2> m_snap_object_map; + + void send_load_map(); + void send_remove_snapshot(); + void send_invalidate_next_map(); + void send_remove_map(); + + void compute_next_snap_id(); + void update_object_map(); +}; + +} // namespace object_map +} // namespace librbd + +#endif // CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_REMOVE_REQUEST_H diff --git a/src/librbd/object_map/SnapshotRollbackRequest.cc b/src/librbd/object_map/SnapshotRollbackRequest.cc new file mode 100644 index 0000000000000..8f5625587eb5e --- /dev/null +++ b/src/librbd/object_map/SnapshotRollbackRequest.cc @@ -0,0 +1,129 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/object_map/SnapshotRollbackRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" +#include "librbd/object_map/InvalidateRequest.h" +#include "cls/lock/cls_lock_client.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::object_map::SnapshotRollbackRequest: " + +namespace librbd { +namespace object_map { + +namespace { + +std::ostream& operator<<(std::ostream& os, + const SnapshotRollbackRequest::State& state) { + switch(state) { + case SnapshotRollbackRequest::STATE_READ_MAP: + os << "READ_MAP"; + break; + case SnapshotRollbackRequest::STATE_INVALIDATE_MAP: + os << "INVALIDATE_MAP"; + break; + case SnapshotRollbackRequest::STATE_WRITE_MAP: + os << "WRITE_MAP"; + break; + default: + os << "UNKNOWN (" << static_cast(state) << ")"; + break; + } + return os; +} + +} // anonymous namespace + +void SnapshotRollbackRequest::send() { + send_read_map(); +} + +bool SnapshotRollbackRequest::should_complete(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << ": state=" << m_state << ", " + << "r=" << r << dendl; + if (r < 0 && m_ret_val == 0) { + m_ret_val = r; + } + + bool finished = false; + switch (m_state) { + case STATE_READ_MAP: + if (r < 0) { + // invalidate the snapshot object map + send_invalidate_map(); + } else { + send_write_map(); + } + break; + case STATE_INVALIDATE_MAP: + // invalidate the HEAD object map as well + finished = Request::should_complete(m_ret_val); + break; + case STATE_WRITE_MAP: + finished = Request::should_complete(r); + break; + default: + assert(false); + break; + } + return finished; +} + +void SnapshotRollbackRequest::send_read_map() { + std::string snap_oid(ObjectMap::object_map_name(m_image_ctx.id, m_snap_id)); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << ": snap_oid=" << snap_oid + << dendl; + m_state = STATE_READ_MAP; + + librados::ObjectReadOperation op; + op.read(0, 0, NULL, NULL); + + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(snap_oid, rados_completion, &op, + &m_read_bl); + assert(r == 0); + rados_completion->release(); +} + +void SnapshotRollbackRequest::send_write_map() { + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_WRITE_MAP; + + librados::ObjectWriteOperation op; + rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", ""); + op.write_full(m_read_bl); + + std::string snap_oid(ObjectMap::object_map_name(m_image_ctx.id, CEPH_NOSNAP)); + librados::AioCompletion *rados_completion = create_callback_completion(); + int r = m_image_ctx.md_ctx.aio_operate(snap_oid, rados_completion, &op); + assert(r == 0); + rados_completion->release(); +} + +void SnapshotRollbackRequest::send_invalidate_map() { + RWLock::RLocker owner_locker(m_image_ctx.owner_lock); + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_INVALIDATE_MAP; + + InvalidateRequest *req = new InvalidateRequest(m_image_ctx, m_snap_id, false, + create_callback_context()); + req->send(); +} + +} // namespace object_map +} // namespace librbd diff --git a/src/librbd/object_map/SnapshotRollbackRequest.h b/src/librbd/object_map/SnapshotRollbackRequest.h new file mode 100644 index 0000000000000..471716653825d --- /dev/null +++ b/src/librbd/object_map/SnapshotRollbackRequest.h @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_ROLLBACK_REQUEST_H +#define CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_ROLLBACK_REQUEST_H + +#include "include/int_types.h" +#include "librbd/object_map/Request.h" + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace object_map { + +class SnapshotRollbackRequest : public Request { +public: + /** + * Snapshot rollback goes through the following state machine: + * + * @verbatim + * + * + * | + * v (error) + * STATE_READ_MAP * * * * > STATE_INVALIDATE_MAP + * | | + * v v + * STATE_WRITE_MAP -------> + * + * @endverbatim + * + * If an error occurs within the READ_MAP state, the associated snapshot's + * object map will be flagged as invalid. Otherwise, an error from any state + * will result in the HEAD object map being flagged as invalid via the base + * class. + */ + enum State { + STATE_READ_MAP, + STATE_INVALIDATE_MAP, + STATE_WRITE_MAP + }; + + SnapshotRollbackRequest(ImageCtx &image_ctx, uint64_t snap_id, + Context *on_finish) + : Request(image_ctx, CEPH_NOSNAP, on_finish), + m_snap_id(snap_id), m_ret_val(0) { + assert(snap_id != CEPH_NOSNAP); + } + + virtual void send(); + +protected: + virtual bool should_complete(int r); + + virtual void finish() { + } + +private: + State m_state; + uint64_t m_snap_id; + int m_ret_val; + + bufferlist m_read_bl; + + void send_read_map(); + void send_invalidate_map(); + void send_write_map(); + +}; + +} // namespace object_map +} // namespace librbd + +#endif // CEPH_LIBRBD_OBJECT_MAP_SNAPSHOT_ROLLBACK_REQUEST_H diff --git a/src/librbd/operation/SnapshotCreateRequest.cc b/src/librbd/operation/SnapshotCreateRequest.cc index 2ee41c37f1e14..a046c0812353f 100644 --- a/src/librbd/operation/SnapshotCreateRequest.cc +++ b/src/librbd/operation/SnapshotCreateRequest.cc @@ -7,6 +7,7 @@ #include "librbd/AioImageRequestWQ.h" #include "librbd/ImageCtx.h" #include "librbd/ImageWatcher.h" +#include "librbd/ObjectMap.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -95,7 +96,7 @@ bool SnapshotCreateRequest::should_complete(int r) { break; case STATE_CREATE_SNAP: if (r == 0) { - m_snap_created = true; + update_snap_context(); finished = send_create_object_map(); } else { assert(r == -ESTALE); @@ -111,7 +112,6 @@ bool SnapshotCreateRequest::should_complete(int r) { } if (finished) { - update_snap_context(); resume_aio(); resume_requests(); } @@ -181,15 +181,21 @@ void SnapshotCreateRequest::send_allocate_snap_id() { void SnapshotCreateRequest::send_create_snap() { assert(m_image_ctx.owner_lock.is_locked()); + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::RLocker parent_locker(m_image_ctx.parent_lock); CephContext *cct = m_image_ctx.cct; ldout(cct, 5) << this << " " << __func__ << dendl; m_state = STATE_CREATE_SNAP; // should have been canceled prior to releasing lock - assert(!m_image_ctx.image_watcher->is_lock_supported() || + assert(!m_image_ctx.image_watcher->is_lock_supported(m_image_ctx.snap_lock) || m_image_ctx.image_watcher->is_lock_owner()); + // save current size / parent info for creating snapshot record in ImageCtx + m_size = m_image_ctx.size; + m_parent_info = m_image_ctx.parent_md; + librados::ObjectWriteOperation op; if (m_image_ctx.old_format) { cls_client::old_snapshot_add(&op, m_snap_id, m_snap_name); @@ -208,7 +214,20 @@ void SnapshotCreateRequest::send_create_snap() { } bool SnapshotCreateRequest::send_create_object_map() { - // TODO add object map support + assert(m_image_ctx.owner_lock.is_locked()); + + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::RLocker object_map_lock(m_image_ctx.object_map_lock); + if (m_image_ctx.object_map.enabled(m_image_ctx.object_map_lock)) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_CREATE_OBJECT_MAP; + + m_image_ctx.object_map.snapshot_add(m_snap_id, create_callback_context()); + return false; + } + } return true; } @@ -252,9 +271,14 @@ void SnapshotCreateRequest::resume_requests() { void SnapshotCreateRequest::update_snap_context() { assert(m_image_ctx.owner_lock.is_locked()); + m_snap_created = true; RWLock::WLocker snap_locker(m_image_ctx.snap_lock); - if (m_image_ctx.old_format || !m_snap_created) { + if (m_image_ctx.old_format) { + return; + } + + if (m_image_ctx.get_snap_info(m_snap_id) != NULL) { return; } @@ -265,6 +289,10 @@ void SnapshotCreateRequest::update_snap_context() { assert(!m_image_ctx.image_watcher->is_lock_supported(m_image_ctx.snap_lock) || m_image_ctx.image_watcher->is_lock_owner()); + // immediately add a reference to the new snapshot + m_image_ctx.add_snap(m_snap_name, m_snap_id, m_size, m_parent_info, + RBD_PROTECTION_STATUS_UNPROTECTED, 0); + // immediately start using the new snap context if we // own the exclusive lock std::vector snaps; diff --git a/src/librbd/operation/SnapshotCreateRequest.h b/src/librbd/operation/SnapshotCreateRequest.h index 1e9d526311f92..7031a43c46f9f 100644 --- a/src/librbd/operation/SnapshotCreateRequest.h +++ b/src/librbd/operation/SnapshotCreateRequest.h @@ -5,6 +5,7 @@ #define CEPH_LIBRBD_OPERATION_SNAPSHOT_CREATE_REQUEST_H #include "librbd/operation/Request.h" +#include "librbd/parent_types.h" #include #include @@ -93,6 +94,9 @@ private: uint64_t m_snap_id; bool m_snap_created; + uint64_t m_size; + parent_info m_parent_info; + int filter_state_return_code(int r) const { if (m_state == STATE_CREATE_SNAP && r == -ESTALE) { return 0; diff --git a/src/librbd/operation/SnapshotRemoveRequest.cc b/src/librbd/operation/SnapshotRemoveRequest.cc index 794a9770b60e3..e8e76b09d86a0 100644 --- a/src/librbd/operation/SnapshotRemoveRequest.cc +++ b/src/librbd/operation/SnapshotRemoveRequest.cc @@ -6,6 +6,7 @@ #include "common/errno.h" #include "librbd/ImageCtx.h" #include "librbd/ImageWatcher.h" +#include "librbd/ObjectMap.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -88,7 +89,19 @@ bool SnapshotRemoveRequest::should_complete(int r) { void SnapshotRemoveRequest::send_remove_object_map() { assert(m_image_ctx.owner_lock.is_locked()); - // TODO add object map support + { + RWLock::WLocker snap_locker(m_image_ctx.snap_lock); + RWLock::RLocker object_map_locker(m_image_ctx.object_map_lock); + if (m_image_ctx.object_map.enabled(m_image_ctx.object_map_lock)) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_REMOVE_OBJECT_MAP; + + m_image_ctx.object_map.snapshot_remove( + m_snap_id, create_callback_context()); + return; + } + } send_remove_child(); } diff --git a/src/librbd/operation/SnapshotRollbackRequest.cc b/src/librbd/operation/SnapshotRollbackRequest.cc index 33972eff4e998..bde4a3a5cfb4d 100644 --- a/src/librbd/operation/SnapshotRollbackRequest.cc +++ b/src/librbd/operation/SnapshotRollbackRequest.cc @@ -7,6 +7,7 @@ #include "common/errno.h" #include "librbd/AsyncObjectThrottle.h" #include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" #include "librbd/operation/ResizeRequest.h" #include "osdc/Striper.h" #include @@ -145,7 +146,19 @@ void SnapshotRollbackRequest::send_resize_image() { void SnapshotRollbackRequest::send_rollback_object_map() { assert(m_image_ctx.owner_lock.is_locked()); - // TODO add object map support + { + RWLock::RLocker snap_locker(m_image_ctx.snap_lock); + RWLock::WLocker object_map_lock(m_image_ctx.object_map_lock); + if (m_image_ctx.object_map.enabled(m_image_ctx.object_map_lock)) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 5) << this << " " << __func__ << dendl; + m_state = STATE_ROLLBACK_OBJECT_MAP; + + m_image_ctx.object_map.rollback(m_snap_id, create_callback_context()); + return; + } + } + send_rollback_objects(); } -- 2.39.5