From: J. Eric Ivancich Date: Fri, 12 Oct 2018 14:23:57 +0000 (-0400) Subject: cls: add exclusive ephemeral locks that auto-clean X-Git-Tag: v12.2.11~115^2~14 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=998a9d2e275c5ad917687ab8eb8a71c092e06502;p=ceph.git cls: add exclusive ephemeral locks that auto-clean Add a new type of cls lock -- exclusive ephemeral for which the object only exists to represent the lock and for which the object should be deleted at unlock. This is to prevent the accumulation of unneeded objects in the cluster by automatically cleaning them up. Signed-off-by: J. Eric Ivancich (cherry picked from commit a289f2d8654cf4b865430465b87299d3618b41c8) --- diff --git a/src/cls/lock/cls_lock.cc b/src/cls/lock/cls_lock.cc index f8081bf7eafd..1dab0dd72aca 100644 --- a/src/cls/lock/cls_lock.cc +++ b/src/cls/lock/cls_lock.cc @@ -34,7 +34,18 @@ CLS_NAME(lock) #define LOCK_PREFIX "lock." -static int read_lock(cls_method_context_t hctx, const string& name, lock_info_t *lock) +static int clean_lock(cls_method_context_t hctx) +{ + int r = cls_cxx_remove(hctx); + if (r < 0) + return r; + + return 0; +} + +static int read_lock(cls_method_context_t hctx, + const string& name, + lock_info_t *lock) { bufferlist bl; string key = LOCK_PREFIX; @@ -67,16 +78,20 @@ static int read_lock(cls_method_context_t hctx, const string& name, lock_info_t map::iterator iter = lock->lockers.begin(); while (iter != lock->lockers.end()) { - map::iterator next = iter; - ++next; - struct locker_info_t& info = iter->second; if (!info.expiration.is_zero() && info.expiration < now) { CLS_LOG(20, "expiring locker"); - lock->lockers.erase(iter); + iter = lock->lockers.erase(iter); + } else { + ++iter; } + } - iter = next; + if (lock->lockers.empty() && cls_lock_is_ephemeral(lock->lock_type)) { + r = clean_lock(hctx); + if (r < 0) { + CLS_ERR("error, on read, cleaning lock object %s", cpp_strerror(r).c_str()); + } } return 0; @@ -121,15 +136,17 @@ static int lock_obj(cls_method_context_t hctx, const string& cookie, const string& tag) { - bool exclusive = lock_type == LOCK_EXCLUSIVE; + bool exclusive = cls_lock_is_exclusive(lock_type); lock_info_t linfo; bool fail_if_exists = (flags & LOCK_FLAG_MAY_RENEW) == 0; bool fail_if_does_not_exist = flags & LOCK_FLAG_MUST_RENEW; - CLS_LOG(20, "requested lock_type=%s fail_if_exists=%d fail_if_does_not_exist=%d", cls_lock_type_str(lock_type), fail_if_exists, fail_if_does_not_exist); - if (lock_type != LOCK_EXCLUSIVE && - lock_type != LOCK_SHARED) + CLS_LOG(20, + "requested lock_type=%s fail_if_exists=%d fail_if_does_not_exist=%d", + cls_lock_type_str(lock_type), fail_if_exists, fail_if_does_not_exist); + if (!cls_lock_is_valid(lock_type)) { return -EINVAL; + } if (name.empty()) return -EINVAL; @@ -147,6 +164,7 @@ static int lock_obj(cls_method_context_t hctx, CLS_ERR("Could not read lock info: %s", cpp_strerror(r).c_str()); return r; } + map& lockers = linfo.lockers; map::iterator iter; @@ -245,9 +263,9 @@ static int lock_op(cls_method_context_t hctx, * entity or cookie is wrong), or -errno on other error. */ static int remove_lock(cls_method_context_t hctx, - const string& name, - entity_name_t& locker, - const string& cookie) + const string& name, + entity_name_t& locker, + const string& cookie) { // get current lockers lock_info_t linfo; @@ -267,7 +285,12 @@ static int remove_lock(cls_method_context_t hctx, } lockers.erase(iter); - r = write_lock(hctx, name, linfo); + if (cls_lock_is_ephemeral(linfo.lock_type)) { + ceph_assert(lockers.empty()); + r = clean_lock(hctx); + } else { + r = write_lock(hctx, name, linfo); + } return r; } @@ -311,7 +334,7 @@ static int unlock_op(cls_method_context_t hctx, * is wrong), or -errno on other (unexpected) error. */ static int break_lock(cls_method_context_t hctx, - bufferlist *in, bufferlist *out) + bufferlist *in, bufferlist *out) { CLS_LOG(20, "break_lock"); cls_lock_break_op op; diff --git a/src/cls/lock/cls_lock_client.cc b/src/cls/lock/cls_lock_client.cc index 3a3cef367947..776fe4889e86 100644 --- a/src/cls/lock/cls_lock_client.cc +++ b/src/cls/lock/cls_lock_client.cc @@ -208,14 +208,19 @@ namespace rados { rados_op->exec("lock", "set_cookie", in); } + void Lock::assert_locked_shared(ObjectOperation *op) + { + assert_locked(op, name, LOCK_SHARED, cookie, tag); + } + void Lock::assert_locked_exclusive(ObjectOperation *op) { assert_locked(op, name, LOCK_EXCLUSIVE, cookie, tag); } - void Lock::assert_locked_shared(ObjectOperation *op) + void Lock::assert_locked_exclusive_ephemeral(ObjectOperation *op) { - assert_locked(op, name, LOCK_SHARED, cookie, tag); + assert_locked(op, name, LOCK_EXCLUSIVE_EPHEMERAL, cookie, tag); } void Lock::lock_shared(ObjectWriteOperation *op) @@ -242,6 +247,18 @@ namespace rados { cookie, tag, description, duration, flags); } + void Lock::lock_exclusive_ephemeral(ObjectWriteOperation *op) + { + lock(op, name, LOCK_EXCLUSIVE_EPHEMERAL, + cookie, tag, description, duration, flags); + } + + int Lock::lock_exclusive_ephemeral(IoCtx *ioctx, const string& oid) + { + return lock(ioctx, oid, name, LOCK_EXCLUSIVE_EPHEMERAL, + cookie, tag, description, duration, flags); + } + void Lock::unlock(ObjectWriteOperation *op) { rados::cls::lock::unlock(op, name, cookie); diff --git a/src/cls/lock/cls_lock_client.h b/src/cls/lock/cls_lock_client.h index e2c21cc9e5b0..0066dc3c0d9f 100644 --- a/src/cls/lock/cls_lock_client.h +++ b/src/cls/lock/cls_lock_client.h @@ -101,6 +101,7 @@ namespace rados { flags &= ~LOCK_FLAG_MAY_RENEW; } } + void set_must_renew(bool renew) { if (renew) { flags |= LOCK_FLAG_MUST_RENEW; @@ -110,18 +111,31 @@ namespace rados { } } - void assert_locked_exclusive(librados::ObjectOperation *rados_op); void assert_locked_shared(librados::ObjectOperation *rados_op); + void assert_locked_exclusive(librados::ObjectOperation *rados_op); + void assert_locked_exclusive_ephemeral(librados::ObjectOperation *rados_op); /* ObjectWriteOperation */ - void lock_exclusive(librados::ObjectWriteOperation *ioctx); void lock_shared(librados::ObjectWriteOperation *ioctx); + void lock_exclusive(librados::ObjectWriteOperation *ioctx); + + // Be careful when using an exclusive ephemeral lock; it is + // intended strictly for cases when a lock object exists + // solely for a lock in a given process and the object is no + // longer needed when the lock is unlocked or expired, as the + // cls back-end will make an effort to delete it. + void lock_exclusive_ephemeral(librados::ObjectWriteOperation *ioctx); void unlock(librados::ObjectWriteOperation *ioctx); - void break_lock(librados::ObjectWriteOperation *ioctx, const entity_name_t& locker); + void break_lock(librados::ObjectWriteOperation *ioctx, + const entity_name_t& locker); /* IoCtx */ - int lock_exclusive(librados::IoCtx *ioctx, const std::string& oid); int lock_shared(librados::IoCtx *ioctx, const std::string& oid); + int lock_exclusive(librados::IoCtx *ioctx, const std::string& oid); + + // NB: see above comment on exclusive ephemeral locks + int lock_exclusive_ephemeral(librados::IoCtx *ioctx, + const std::string& oid); int unlock(librados::IoCtx *ioctx, const std::string& oid); int break_lock(librados::IoCtx *ioctx, const std::string& oid, const entity_name_t& locker); diff --git a/src/cls/lock/cls_lock_types.h b/src/cls/lock/cls_lock_types.h index cc2e5617a0a6..5f44126b4879 100644 --- a/src/cls/lock/cls_lock_types.h +++ b/src/cls/lock/cls_lock_types.h @@ -14,9 +14,10 @@ #define LOCK_FLAG_MUST_RENEW 0x2 /* lock must already be acquired */ enum ClsLockType { - LOCK_NONE = 0, - LOCK_EXCLUSIVE = 1, - LOCK_SHARED = 2, + LOCK_NONE = 0, + LOCK_EXCLUSIVE = 1, + LOCK_SHARED = 2, + LOCK_EXCLUSIVE_EPHEMERAL = 3, /* lock object is removed @ unlock */ }; static inline const char *cls_lock_type_str(ClsLockType type) @@ -28,11 +29,27 @@ static inline const char *cls_lock_type_str(ClsLockType type) return "exclusive"; case LOCK_SHARED: return "shared"; + case LOCK_EXCLUSIVE_EPHEMERAL: + return "exclusive-ephemeral"; default: return ""; } } +inline bool cls_lock_is_exclusive(ClsLockType type) { + return LOCK_EXCLUSIVE == type || LOCK_EXCLUSIVE_EPHEMERAL == type; +} + +inline bool cls_lock_is_ephemeral(ClsLockType type) { + return LOCK_EXCLUSIVE_EPHEMERAL == type; +} + +inline bool cls_lock_is_valid(ClsLockType type) { + return LOCK_SHARED == type || + LOCK_EXCLUSIVE == type || + LOCK_EXCLUSIVE_EPHEMERAL == type; +} + namespace rados { namespace cls { namespace lock { diff --git a/src/test/cls_lock/test_cls_lock.cc b/src/test/cls_lock/test_cls_lock.cc index b8bc51e01a41..37d10a19cbcf 100644 --- a/src/test/cls_lock/test_cls_lock.cc +++ b/src/test/cls_lock/test_cls_lock.cc @@ -457,3 +457,109 @@ TEST(ClsLock, TestRenew) { ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); } + +TEST(ClsLock, TestExclusiveEphemeralBasic) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string oid2 = "foo2"; + string lock_name1 = "mylock1"; + string lock_name2 = "mylock2"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(5, 0)); + + uint64_t size; + time_t mod_time; + + l1.set_may_renew(true); + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + ASSERT_EQ(0, ioctx.stat(oid1, &size, &mod_time)); + sleep(2); + ASSERT_EQ(0, l1.unlock(&ioctx, oid1)); + ASSERT_EQ(-ENOENT, ioctx.stat(oid1, &size, &mod_time)); + + // *********************************************** + + Lock l2(lock_name2); + utime_t lock_duration2(5, 0); + l2.set_duration(utime_t(5, 0)); + + ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid2)); + ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time)); + sleep(2); + ASSERT_EQ(0, l2.unlock(&ioctx, oid2)); + ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} + + +TEST(ClsLock, TestExclusiveEphemeralStealEphemeral) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string lock_name1 = "mylock1"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(3, 0)); + + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(4); + + // l1 is expired, l2 can take; l2 is also exclusive_ephemeral + Lock l2(lock_name1); + l2.set_duration(utime_t(3, 0)); + ASSERT_EQ(0, l2.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(1); + ASSERT_EQ(0, l2.unlock(&ioctx, oid1)); + + // l2 cannot unlock its expired lock + ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} + + +TEST(ClsLock, TestExclusiveEphemeralStealExclusive) { + Rados cluster; + std::string pool_name = get_temp_pool_name(); + ASSERT_EQ("", create_one_pool_pp(pool_name, cluster)); + IoCtx ioctx; + cluster.ioctx_create(pool_name.c_str(), ioctx); + + bufferlist bl; + + string oid1 = "foo1"; + string lock_name1 = "mylock1"; + + Lock l1(lock_name1); + l1.set_duration(utime_t(3, 0)); + + ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1)); + sleep(4); + + // l1 is expired, l2 can take; l2 is exclusive (but not ephemeral) + Lock l2(lock_name1); + l2.set_duration(utime_t(3, 0)); + ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid1)); + sleep(1); + ASSERT_EQ(0, l2.unlock(&ioctx, oid1)); + + // l2 cannot unlock its expired lock + ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1)); + + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); +} diff --git a/src/test/librados_test_stub/LibradosTestStub.cc b/src/test/librados_test_stub/LibradosTestStub.cc index c07d4b61bb28..e5e52948116d 100644 --- a/src/test/librados_test_stub/LibradosTestStub.cc +++ b/src/test/librados_test_stub/LibradosTestStub.cc @@ -1140,6 +1140,12 @@ int cls_cxx_create(cls_method_context_t hctx, bool exclusive) { return ctx->io_ctx_impl->create(ctx->oid, exclusive); } +int cls_cxx_remove(cls_method_context_t hctx) { + librados::TestClassHandler::MethodContext *ctx = + reinterpret_cast(hctx); + return ctx->io_ctx_impl->remove(ctx->oid, ctx->io_ctx_impl->get_snap_context()); +} + int cls_get_request_origin(cls_method_context_t hctx, entity_inst_t *origin) { librados::TestClassHandler::MethodContext *ctx = reinterpret_cast(hctx);