From 7e3a032224ba0c3c3bbfef2576b33abcd1f8d499 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2015 23:57:37 -0500 Subject: [PATCH] librbd: all object map methods are now async Signed-off-by: Jason Dillaman --- src/librbd/ImageCtx.cc | 25 ++++- src/librbd/ImageCtx.h | 5 +- src/librbd/ObjectMap.cc | 236 +++++----------------------------------- src/librbd/ObjectMap.h | 9 +- 4 files changed, 55 insertions(+), 220 deletions(-) diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index 7da1ad5eb7bde..6cb3979f77936 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -534,10 +534,17 @@ struct C_InvalidateCache : public Context { return 0; } - bool ImageCtx::test_features(uint64_t test_features) const + bool ImageCtx::test_features(uint64_t features) const { RWLock::RLocker l(snap_lock); - return ((features & test_features) == test_features); + return test_features(features, snap_lock); + } + + bool ImageCtx::test_features(uint64_t in_features, + const RWLock &in_snap_lock) const + { + assert(snap_lock.is_locked()); + return ((features & in_features) == in_features); } int ImageCtx::get_flags(librados::snap_t _snap_id, uint64_t *_flags) const @@ -555,12 +562,18 @@ struct C_InvalidateCache : public Context { return -ENOENT; } - bool ImageCtx::test_flags(uint64_t test_flags) const + bool ImageCtx::test_flags(uint64_t flags) const { RWLock::RLocker l(snap_lock); + return test_flags(flags, snap_lock); + } + + bool ImageCtx::test_flags(uint64_t flags, const RWLock &in_snap_lock) const + { + assert(snap_lock.is_locked()); uint64_t snap_flags; get_flags(snap_id, &snap_flags); - return ((snap_flags & test_flags) == test_flags); + return ((snap_flags & flags) == flags); } int ImageCtx::update_flags(snap_t in_snap_id, uint64_t flag, bool enabled) @@ -1004,8 +1017,8 @@ struct C_InvalidateCache : public Context { ASSIGN_OPTION(journal_pool); } - ObjectMap *ImageCtx::create_object_map() { - return new ObjectMap(*this); + ObjectMap *ImageCtx::create_object_map(uint64_t snap_id) { + return new ObjectMap(*this, snap_id); } Journal *ImageCtx::create_journal() { diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index a64ba404e5206..5845f6425e00d 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -228,8 +228,11 @@ namespace librbd { void rm_snap(std::string in_snap_name, librados::snap_t id); uint64_t get_image_size(librados::snap_t in_snap_id) const; bool test_features(uint64_t test_features) const; + bool test_features(uint64_t test_features, + const RWLock &in_snap_lock) const; int get_flags(librados::snap_t in_snap_id, uint64_t *flags) const; bool test_flags(uint64_t test_flags) const; + bool test_flags(uint64_t test_flags, const RWLock &in_snap_lock) const; int update_flags(librados::snap_t in_snap_id, uint64_t flag, bool enabled); const parent_info* get_parent_info(librados::snap_t in_snap_id) const; @@ -269,7 +272,7 @@ namespace librbd { void apply_metadata_confs(); - ObjectMap *create_object_map(); + ObjectMap *create_object_map(uint64_t snap_id); Journal *create_journal(); void open_journal(); // TODO remove diff --git a/src/librbd/ObjectMap.cc b/src/librbd/ObjectMap.cc index c023398158b91..4f3f2f4958617 100644 --- a/src/librbd/ObjectMap.cc +++ b/src/librbd/ObjectMap.cc @@ -27,25 +27,8 @@ namespace librbd { -namespace { - -struct C_ApplyRefresh : public Context { - object_map::RefreshRequest<> *request; - C_ApplyRefresh(object_map::RefreshRequest<> *request) : request(request) { - } - virtual void finish(int r) { - if (r < 0) { - request->apply(); - } else { - request->discard(); - } - } -}; - -} // anonymous namespace - -ObjectMap::ObjectMap(ImageCtx &image_ctx) - : m_image_ctx(image_ctx), m_snap_id(CEPH_NOSNAP) +ObjectMap::ObjectMap(ImageCtx &image_ctx, uint64_t snap_id) + : m_image_ctx(image_ctx), m_snap_id(snap_id) { } @@ -79,101 +62,15 @@ uint8_t ObjectMap::operator[](uint64_t object_no) const return m_object_map[object_no]; } -int ObjectMap::lock() -{ - if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)) { - return 0; - } - - bool broke_lock = false; - CephContext *cct = m_image_ctx.cct; - std::string oid(object_map_name(m_image_ctx.id, CEPH_NOSNAP)); - while (true) { - int r; - ldout(cct, 10) << &m_image_ctx << " locking object map: " - << oid << dendl; - r = rados::cls::lock::lock(&m_image_ctx.md_ctx, oid, - RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", "", "", - utime_t(), 0); - if (r == 0) { - break; - } else if (broke_lock || r != -EBUSY) { - lderr(cct) << "failed to lock object map: " << cpp_strerror(r) << dendl; - return r; - } - - typedef std::map lockers_t; - lockers_t lockers; - ClsLockType lock_type; - std::string lock_tag; - r = rados::cls::lock::get_lock_info(&m_image_ctx.md_ctx, oid, - RBD_LOCK_NAME, &lockers, - &lock_type, &lock_tag); - if (r == -ENOENT) { - continue; - } else if (r < 0) { - lderr(cct) << "failed to list object map locks: " << cpp_strerror(r) - << dendl; - return r; - } - - ldout(cct, 10) << "breaking current object map lock" << dendl; - for (lockers_t::iterator it = lockers.begin(); - it != lockers.end(); ++it) { - const rados::cls::lock::locker_id_t &locker = it->first; - r = rados::cls::lock::break_lock(&m_image_ctx.md_ctx, oid, - RBD_LOCK_NAME, locker.cookie, - locker.locker); - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to break object map lock: " << cpp_strerror(r) - << dendl; - return r; - } - } - - - - broke_lock = true; - } - return 0; -} - -void ObjectMap::lock(Context *on_finish) { - object_map::LockRequest<> *req = new object_map::LockRequest<>( - m_image_ctx, on_finish); - req->send(); -} - -int ObjectMap::unlock() -{ - if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)) { - return 0; - } - - std::string oid = object_map_name(m_image_ctx.id, CEPH_NOSNAP); - ldout(m_image_ctx.cct, 10) << &m_image_ctx << " unlocking object map: " - << oid << dendl; - int r = rados::cls::lock::unlock(&m_image_ctx.md_ctx, oid, - RBD_LOCK_NAME, ""); - if (r < 0 && r != -ENOENT) { - lderr(m_image_ctx.cct) << "failed to release object map lock: " - << cpp_strerror(r) << dendl; - } - return r; -} - -void ObjectMap::unlock(Context *on_finish) { - object_map::UnlockRequest<> *req = new object_map::UnlockRequest<>( - m_image_ctx, on_finish); - req->send(); -} - bool ObjectMap::object_may_exist(uint64_t object_no) const { + assert(m_image_ctx.snap_lock.is_locked()); + // Fall back to default logic if object map is disabled or invalid - if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP) || - m_image_ctx.test_flags(RBD_FLAG_OBJECT_MAP_INVALID)) { + if (!m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP, + m_image_ctx.snap_lock) || + m_image_ctx.test_flags(RBD_FLAG_OBJECT_MAP_INVALID, + m_image_ctx.snap_lock)) { return true; } @@ -186,83 +83,24 @@ bool ObjectMap::object_may_exist(uint64_t object_no) const return exists; } -void ObjectMap::refresh(uint64_t snap_id) -{ - assert(m_image_ctx.snap_lock.is_wlocked()); - RWLock::WLocker l(m_image_ctx.object_map_lock); - m_snap_id = snap_id; - - CephContext *cct = m_image_ctx.cct; - std::string oid(object_map_name(m_image_ctx.id, snap_id)); - ldout(cct, 10) << &m_image_ctx << " refreshing object map: " - << oid << dendl; - - uint64_t num_objs = Striper::get_num_objects( - m_image_ctx.layout, m_image_ctx.get_image_size(snap_id)); - - int r = cls_client::object_map_load(&m_image_ctx.md_ctx, oid, - &m_object_map); - if (r == -EINVAL) { - // object map is corrupt on-disk -- clear it and properly size it - // so future IO can keep the object map in sync - invalidate(snap_id, false); - - librados::ObjectWriteOperation op; - if (snap_id == CEPH_NOSNAP) { - rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", - ""); - } - op.truncate(0); - cls_client::object_map_resize(&op, num_objs, OBJECT_NONEXISTENT); - - r = m_image_ctx.md_ctx.operate(oid, &op); - if (r == 0) { - m_object_map.clear(); - object_map::ResizeRequest::resize(&m_object_map, num_objs, - OBJECT_NONEXISTENT); - } - } - if (r < 0) { - lderr(cct) << "error refreshing object map: " << cpp_strerror(r) - << dendl; - invalidate(snap_id, false); - m_object_map.clear(); - return; - } - - ldout(cct, 20) << "refreshed object map: num_objs=" << m_object_map.size() - << dendl; - - if (m_object_map.size() < num_objs) { - lderr(cct) << "object map smaller than current object count: " - << m_object_map.size() << " != " << num_objs << dendl; - invalidate(snap_id, false); - - // correct the size issue so future IO can keep the object map in sync - librados::ObjectWriteOperation op; - if (snap_id == CEPH_NOSNAP) { - rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME, LOCK_EXCLUSIVE, "", - ""); - } - cls_client::object_map_resize(&op, num_objs, OBJECT_NONEXISTENT); +void ObjectMap::open(Context *on_finish) { + object_map::RefreshRequest<> *req = new object_map::RefreshRequest<>( + m_image_ctx, &m_object_map, m_snap_id, on_finish); + req->send(); +} - r = m_image_ctx.md_ctx.operate(oid, &op); - if (r == 0) { - object_map::ResizeRequest::resize(&m_object_map, num_objs, - OBJECT_NONEXISTENT); - } - } else if (m_object_map.size() > num_objs) { - // resize op might have been interrupted - ldout(cct, 1) << "object map larger than current object count: " - << m_object_map.size() << " != " << num_objs << dendl; - } +void ObjectMap::lock(Context *on_finish) { + assert(m_snap_id == CEPH_NOSNAP); + object_map::LockRequest<> *req = new object_map::LockRequest<>( + m_image_ctx, on_finish); + req->send(); } -Context* ObjectMap::refresh(uint64_t snap_id, Context *on_finish) { - object_map::RefreshRequest<> *req = new object_map::RefreshRequest<>( - m_image_ctx, &m_object_map, snap_id, on_finish); +void ObjectMap::unlock(Context *on_finish) { + assert(m_snap_id == CEPH_NOSNAP); + object_map::UnlockRequest<> *req = new object_map::UnlockRequest<>( + m_image_ctx, on_finish); req->send(); - return new C_ApplyRefresh(req); } void ObjectMap::rollback(uint64_t snap_id, Context *on_finish) { @@ -277,6 +115,7 @@ void ObjectMap::rollback(uint64_t snap_id, Context *on_finish) { void ObjectMap::snapshot_add(uint64_t snap_id, Context *on_finish) { assert(m_image_ctx.snap_lock.is_locked()); assert((m_image_ctx.features & RBD_FEATURE_OBJECT_MAP) != 0); + assert(snap_id != CEPH_NOSNAP); object_map::SnapshotCreateRequest *req = new object_map::SnapshotCreateRequest(m_image_ctx, &m_object_map, snap_id, @@ -295,10 +134,11 @@ void ObjectMap::snapshot_remove(uint64_t snap_id, Context *on_finish) { req->send(); } -void ObjectMap::aio_save(Context *on_finish) -{ - assert(m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)); +void ObjectMap::aio_save(Context *on_finish) { assert(m_image_ctx.owner_lock.is_locked()); + assert(m_image_ctx.snap_lock.is_locked()); + assert(m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP, + m_image_ctx.snap_lock)); RWLock::RLocker object_map_locker(m_image_ctx.object_map_lock); librados::ObjectWriteOperation op; @@ -317,8 +157,10 @@ void ObjectMap::aio_save(Context *on_finish) void ObjectMap::aio_resize(uint64_t new_size, uint8_t default_object_state, Context *on_finish) { - assert(m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP)); assert(m_image_ctx.owner_lock.is_locked()); + assert(m_image_ctx.snap_lock.is_locked()); + assert(m_image_ctx.test_features(RBD_FEATURE_OBJECT_MAP, + m_image_ctx.snap_lock)); assert(m_image_ctx.image_watcher != NULL); assert(m_image_ctx.exclusive_lock == nullptr || m_image_ctx.exclusive_lock->is_lock_owner()); @@ -386,22 +228,4 @@ void ObjectMap::aio_update(uint64_t snap_id, uint64_t start_object_no, req->send(); } -void ObjectMap::invalidate(uint64_t snap_id, bool force) { - assert(m_image_ctx.snap_lock.is_wlocked()); - assert(m_image_ctx.object_map_lock.is_wlocked()); - uint64_t flags; - m_image_ctx.get_flags(snap_id, &flags); - if ((flags & RBD_FLAG_OBJECT_MAP_INVALID) != 0) { - return; - } - - // TODO remove once all methods are async - C_SaferCond cond_ctx; - object_map::InvalidateRequest<> *req = new object_map::InvalidateRequest<>( - m_image_ctx, m_snap_id, force, &cond_ctx); - req->send(); - - cond_ctx.wait(); -} - } // namespace librbd diff --git a/src/librbd/ObjectMap.h b/src/librbd/ObjectMap.h index 39de022c02f57..5253c51aa3546 100644 --- a/src/librbd/ObjectMap.h +++ b/src/librbd/ObjectMap.h @@ -18,8 +18,7 @@ class ImageCtx; class ObjectMap { public: - - ObjectMap(ImageCtx &image_ctx); + ObjectMap(ImageCtx &image_ctx, uint64_t snap_id); static int remove(librados::IoCtx &io_ctx, const std::string &image_id); static std::string object_map_name(const std::string &image_id, @@ -31,9 +30,8 @@ public: return m_object_map.size(); } - int lock(); // TODO remove + void open(Context *on_finish); void lock(Context *on_finish); - int unlock(); // TODO remove void unlock(Context *on_finish); bool object_may_exist(uint64_t object_no) const; @@ -54,8 +52,6 @@ public: const boost::optional ¤t_state, Context *on_finish); - void refresh(uint64_t snap_id); // TODO remove - Context *refresh(uint64_t snap_id, Context *on_finish); void rollback(uint64_t snap_id, Context *on_finish); void snapshot_add(uint64_t snap_id, Context *on_finish); void snapshot_remove(uint64_t snap_id, Context *on_finish); @@ -65,7 +61,6 @@ private: ceph::BitVector<2> m_object_map; uint64_t m_snap_id; - void invalidate(uint64_t snap_id, bool force); }; } // namespace librbd -- 2.39.5