From 998a9d2e275c5ad917687ab8eb8a71c092e06502 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Fri, 12 Oct 2018 10:23:57 -0400 Subject: [PATCH] cls: add exclusive ephemeral locks that auto-clean Add a new type of cls lock -- exclusive ephemeral for which the object only exists to represent the lock and for which the object should be deleted at unlock. This is to prevent the accumulation of unneeded objects in the cluster by automatically cleaning them up. Signed-off-by: J. Eric Ivancich (cherry picked from commit a289f2d8654cf4b865430465b87299d3618b41c8) --- src/cls/lock/cls_lock.cc | 53 ++++++--- src/cls/lock/cls_lock_client.cc | 21 +++- src/cls/lock/cls_lock_client.h | 22 +++- src/cls/lock/cls_lock_types.h | 23 +++- src/test/cls_lock/test_cls_lock.cc | 106 ++++++++++++++++++ .../librados_test_stub/LibradosTestStub.cc | 6 + 6 files changed, 207 insertions(+), 24 deletions(-) diff --git a/src/cls/lock/cls_lock.cc b/src/cls/lock/cls_lock.cc index f8081bf7eafd4..1dab0dd72aca1 100644 --- a/src/cls/lock/cls_lock.cc +++ b/src/cls/lock/cls_lock.cc @@ -34,7 +34,18 @@ CLS_NAME(lock) #define LOCK_PREFIX "lock." -static int read_lock(cls_method_context_t hctx, const string& name, lock_info_t *lock) +static int clean_lock(cls_method_context_t hctx) +{ + int r = cls_cxx_remove(hctx); + if (r < 0) + return r; + + return 0; +} + +static int read_lock(cls_method_context_t hctx, + const string& name, + lock_info_t *lock) { bufferlist bl; string key = LOCK_PREFIX; @@ -67,16 +78,20 @@ static int read_lock(cls_method_context_t hctx, const string& name, lock_info_t map::iterator iter = lock->lockers.begin(); while (iter != lock->lockers.end()) { - map::iterator next = iter; - ++next; - struct locker_info_t& info = iter->second; if (!info.expiration.is_zero() && info.expiration < now) { CLS_LOG(20, "expiring locker"); - lock->lockers.erase(iter); + iter = lock->lockers.erase(iter); + } else { + ++iter; } + } - iter = next; + if (lock->lockers.empty() && cls_lock_is_ephemeral(lock->lock_type)) { + r = clean_lock(hctx); + if (r < 0) { + CLS_ERR("error, on read, cleaning lock object %s", cpp_strerror(r).c_str()); + } } return 0; @@ -121,15 +136,17 @@ static int lock_obj(cls_method_context_t hctx, const string& cookie, const string& tag) { - bool exclusive = lock_type == LOCK_EXCLUSIVE; + bool exclusive = cls_lock_is_exclusive(lock_type); lock_info_t linfo; bool fail_if_exists = (flags & LOCK_FLAG_MAY_RENEW) == 0; bool fail_if_does_not_exist = flags & LOCK_FLAG_MUST_RENEW; - CLS_LOG(20, "requested lock_type=%s fail_if_exists=%d fail_if_does_not_exist=%d", cls_lock_type_str(lock_type), fail_if_exists, fail_if_does_not_exist); - if (lock_type != LOCK_EXCLUSIVE && - lock_type != LOCK_SHARED) + CLS_LOG(20, + "requested lock_type=%s fail_if_exists=%d fail_if_does_not_exist=%d", + cls_lock_type_str(lock_type), fail_if_exists, fail_if_does_not_exist); + if (!cls_lock_is_valid(lock_type)) { return -EINVAL; + } if (name.empty()) return -EINVAL; @@ -147,6 +164,7 @@ static int lock_obj(cls_method_context_t hctx, CLS_ERR("Could not read lock info: %s", cpp_strerror(r).c_str()); return r; } + map& lockers = linfo.lockers; map::iterator iter; @@ -245,9 +263,9 @@ static int lock_op(cls_method_context_t hctx, * entity or cookie is wrong), or -errno on other error. */ static int remove_lock(cls_method_context_t hctx, - const string& name, - entity_name_t& locker, - const string& cookie) + const string& name, + entity_name_t& locker, + const string& cookie) { // get current lockers lock_info_t linfo; @@ -267,7 +285,12 @@ static int remove_lock(cls_method_context_t hctx, } lockers.erase(iter); - r = write_lock(hctx, name, linfo); + if (cls_lock_is_ephemeral(linfo.lock_type)) { + ceph_assert(lockers.empty()); + r = clean_lock(hctx); + } else { + r = write_lock(hctx, name, linfo); + } return r; } @@ -311,7 +334,7 @@ static int unlock_op(cls_method_context_t hctx, * is wrong), or -errno on other (unexpected) error. */ static int break_lock(cls_method_context_t hctx, - bufferlist *in, bufferlist *out) + bufferlist *in, bufferlist *out) { CLS_LOG(20, "break_lock"); cls_lock_break_op op; diff --git a/src/cls/lock/cls_lock_client.cc b/src/cls/lock/cls_lock_client.cc index 3a3cef3679478..776fe4889e86f 100644 --- a/src/cls/lock/cls_lock_client.cc +++ b/src/cls/lock/cls_lock_client.cc @@ -208,14 +208,19 @@ namespace rados { rados_op->exec("lock", "set_cookie", in); } + void Lock::assert_locked_shared(ObjectOperation *op) + { + assert_locked(op, name, LOCK_SHARED, cookie, tag); + } + void Lock::assert_locked_exclusive(ObjectOperation *op) { assert_locked(op, name, LOCK_EXCLUSIVE, cookie, tag); } - void Lock::assert_locked_shared(ObjectOperation *op) + void Lock::assert_locked_exclusive_ephemeral(ObjectOperation *op) { - assert_locked(op, name, LOCK_SHARED, cookie, tag); + assert_locked(op, name, LOCK_EXCLUSIVE_EPHEMERAL, cookie, tag); } void Lock::lock_shared(ObjectWriteOperation *op) @@ -242,6 +247,18 @@ namespace rados { cookie, tag, description, duration, flags); } + void Lock::lock_exclusive_ephemeral(ObjectWriteOperation *op) + { + lock(op, name, LOCK_EXCLUSIVE_EPHEMERAL, + cookie, tag, description, duration, flags); + } + + int Lock::lock_exclusive_ephemeral(IoCtx *ioctx, const string& oid) + { + return lock(ioctx, oid, name, LOCK_EXCLUSIVE_EPHEMERAL, + cookie, tag, description, duration, flags); + } + void Lock::unlock(ObjectWriteOperation *op) { rados::cls::lock::unlock(op, name, cookie); diff --git a/src/cls/lock/cls_lock_client.h b/src/cls/lock/cls_lock_client.h index e2c21cc9e5b05..0066dc3c0d9f1 100644 --- a/src/cls/lock/cls_lock_client.h +++ b/src/cls/lock/cls_lock_client.h @@ -101,6 +101,7 @@ namespace rados { flags &= ~LOCK_FLAG_MAY_RENEW; } } + void set_must_renew(bool renew) { if (renew) { flags |= LOCK_FLAG_MUST_RENEW; @@ -110,18 +111,31 @@ namespace rados { } } - void assert_locked_exclusive(librados::ObjectOperation *rados_op); void assert_locked_shared(librados::ObjectOperation *rados_op); + void assert_locked_exclusive(librados::ObjectOperation *rados_op); + void assert_locked_exclusive_ephemeral(librados::ObjectOperation *rados_op); /* ObjectWriteOperation */ - void lock_exclusive(librados::ObjectWriteOperation *ioctx); void lock_shared(librados::ObjectWriteOperation *ioctx); + void lock_exclusive(librados::ObjectWriteOperation *ioctx); + + // Be careful when using an exclusive ephemeral lock; it is + // intended strictly for cases when a lock object exists + // solely for a lock in a given process and the object is no + // longer needed when the lock is unlocked or expired, as the + // cls back-end will make an effort to delete it. + void lock_exclusive_ephemeral(librados::ObjectWriteOperation *ioctx); void unlock(librados::ObjectWriteOperation *ioctx); - void break_lock(librados::ObjectWriteOperation *ioctx, const entity_name_t& locker); + void break_lock(librados::ObjectWriteOperation *ioctx, + const entity_name_t& locker); /* IoCtx */ - int lock_exclusive(librados::IoCtx *ioctx, const std::string& oid); int lock_shared(librados::IoCtx *ioctx, const std::string& oid); + int lock_exclusive(librados::IoCtx *ioctx, const std::string& oid); + + // NB: see above comment on exclusive ephemeral locks + int lock_exclusive_ephemeral(librados::IoCtx *ioctx, + const std::string& oid); int unlock(librados::IoCtx *ioctx, const std::string& oid); int break_lock(librados::IoCtx *ioctx, const std::string& oid, const entity_name_t& locker); diff --git a/src/cls/lock/cls_lock_types.h b/src/cls/lock/cls_lock_types.h index cc2e5617a0a6c..5f44126b4879d 100644 --- a/src/cls/lock/cls_lock_types.h +++ b/src/cls/lock/cls_lock_types.h @@ -14,9 +14,10 @@ #define LOCK_FLAG_MUST_RENEW 0x2 /* lock must already be acquired */ enum ClsLockType { - LOCK_NONE = 0, - LOCK_EXCLUSIVE = 1, - LOCK_SHARED = 2, + LOCK_NONE = 0, + LOCK_EXCLUSIVE = 1, + LOCK_SHARED = 2, + LOCK_EXCLUSIVE_EPHEMERAL = 3, /* lock object is removed @ unlock */ }; static inline const char *cls_lock_type_str(ClsLockType type) @@ -28,11 +29,27 @@ static inline const char *cls_lock_type_str(ClsLockType type) return "exclusive"; case LOCK_SHARED: return "shared"; + case LOCK_EXCLUSIVE_EPHEMERAL: + return "exclusive-ephemeral"; default: return ""; } } +inline bool cls_lock_is_exclusive(ClsLockType type) { + return LOCK_EXCLUSIVE == type || LOCK_EXCLUSIVE_EPHEMERAL == type; +} + +inline bool cls_lock_is_ephemeral(ClsLockType type) { + return LOCK_EXCLUSIVE_EPHEMERAL == type; +} + +inline bool cls_lock_is_valid(ClsLockType type) { + return LOCK_SHARED == type || + LOCK_EXCLUSIVE == type || + LOCK_EXCLUSIVE_EPHEMERAL == type; +} + namespace rados { namespace cls { namespace lock { diff --git a/src/test/cls_lock/test_cls_lock.cc b/src/test/cls_lock/test_cls_lock.cc index b8bc51e01a414..37d10a19cbcf4 100644 --- a/src/test/cls_lock/test_cls_lock.cc +++ b/src/test/cls_lock/test_cls_lock.cc @@ -457,3 +457,109 @@ TEST(ClsLock, TestRenew) { ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); } + +TEST(ClsLock, TestExclusiveEphemeralBasic) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string oid2 = "foo2"; + string lock_name1 = "mylock1"; + string lock_name2 = "mylock2"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(5, 0)); + + uint64_t size; + time_t mod_time; + + l1.set_may_renew(true); + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + ASSERT_EQ(0, ioctx.stat(oid1, &size, &mod_time)); + sleep(2); + ASSERT_EQ(0, l1.unlock(&ioctx, oid1)); + ASSERT_EQ(-ENOENT, ioctx.stat(oid1, &size, &mod_time)); + + // *********************************************** + + Lock l2(lock_name2); + utime_t lock_duration2(5, 0); + l2.set_duration(utime_t(5, 0)); + + ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid2)); + ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time)); + sleep(2); + ASSERT_EQ(0, l2.unlock(&ioctx, oid2)); + ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} + + +TEST(ClsLock, TestExclusiveEphemeralStealEphemeral) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string lock_name1 = "mylock1"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(3, 0)); + + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(4); + + // l1 is expired, l2 can take; l2 is also exclusive_ephemeral + Lock l2(lock_name1); + l2.set_duration(utime_t(3, 0)); + ASSERT_EQ(0, l2.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(1); + ASSERT_EQ(0, l2.unlock(&ioctx, oid1)); + + // l2 cannot unlock its expired lock + ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} + + +TEST(ClsLock, TestExclusiveEphemeralStealExclusive) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string lock_name1 = "mylock1"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(3, 0)); + + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(4); + + // l1 is expired, l2 can take; l2 is exclusive (but not ephemeral) + Lock l2(lock_name1); + l2.set_duration(utime_t(3, 0)); + ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid1)); + sleep(1); + ASSERT_EQ(0, l2.unlock(&ioctx, oid1)); + + // l2 cannot unlock its expired lock + ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} diff --git a/src/test/librados_test_stub/LibradosTestStub.cc b/src/test/librados_test_stub/LibradosTestStub.cc index c07d4b61bb28f..e5e52948116da 100644 --- a/src/test/librados_test_stub/LibradosTestStub.cc +++ b/src/test/librados_test_stub/LibradosTestStub.cc @@ -1140,6 +1140,12 @@ int cls_cxx_create(cls_method_context_t hctx, bool exclusive) { return ctx->io_ctx_impl->create(ctx->oid, exclusive); } +int cls_cxx_remove(cls_method_context_t hctx) { + librados::TestClassHandler::MethodContext *ctx = + reinterpret_cast(hctx); + return ctx->io_ctx_impl->remove(ctx->oid, ctx->io_ctx_impl->get_snap_context()); +} + int cls_get_request_origin(cls_method_context_t hctx, entity_inst_t *origin) { librados::TestClassHandler::MethodContext *ctx = reinterpret_cast(hctx); -- 2.39.5