From 3a9e6650afc4f220c1f35df1853bcc2e1f7ba356 Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Fri, 31 Aug 2012 17:02:01 -0700 Subject: [PATCH] librbd: use generic cls_lock instead of cls_rbd's locking Update the librbd locking api to make more sense: * Add an optional tag to shared locking * only make shared vs exclusive different functions in the user-visible api * return a list of structs instead of a set of pairs * fix incorrect range checking in the C api * rename locks to lockers to be consistent with the generic locking class * rename other_locker parameter to client, to match the list_lockers usage Fixes: #2952 Signed-off-by: Josh Durgin --- src/Makefile.am | 10 ++- src/include/rbd/librbd.h | 107 ++++++++++++++++++++++++++----- src/include/rbd/librbd.hpp | 16 +++-- src/librbd/ImageCtx.h | 6 +- src/librbd/cls_rbd_client.cc | 21 +++++-- src/librbd/cls_rbd_client.h | 5 +- src/librbd/internal.cc | 118 ++++++++++++++++++++++++++++------- src/librbd/internal.h | 15 +++-- src/librbd/librbd.cc | 102 +++++++++++++++++++----------- src/pybind/rbd.py | 101 ++++++++++++++++++++++++++++++ src/test/pybind/test_rbd.py | 35 +++++++++++ src/test/test_librbd.cc | 72 +++++++++++++++++++++ 12 files changed, 512 insertions(+), 96 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 6deed9580033c..5e1eda1fb2ca0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -357,7 +357,10 @@ librbd_la_SOURCES = \ librbd/internal.cc \ librbd/LibrbdWriteback.cc \ librbd/WatchCtx.cc \ - osdc/ObjectCacher.cc + osdc/ObjectCacher.cc \ + cls/lock/cls_lock_client.cc \ + cls/lock/cls_lock_types.cc \ + cls/lock/cls_lock_ops.cc librbd_la_CFLAGS = ${AM_CFLAGS} librbd_la_CXXFLAGS = ${AM_CXXFLAGS} librbd_la_LIBADD = librados.la @@ -725,7 +728,10 @@ bin_DEBUGPROGRAMS += test_librbd_fsx test_cls_rbd_SOURCES = test/rbd/test_cls_rbd.cc \ test/rados-api/test.cc \ - librbd/cls_rbd_client.cc + librbd/cls_rbd_client.cc \ + cls/lock/cls_lock_client.cc \ + cls/lock/cls_lock_types.cc \ + cls/lock/cls_lock_ops.cc test_cls_rbd_LDADD = librados.la ${UNITTEST_STATIC_LDADD} test_cls_rbd_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} bin_DEBUGPROGRAMS += test_cls_rbd diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 77ca1e3af32bc..45d456e717569 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -157,26 +157,103 @@ int rbd_flatten(rbd_image_t image); ssize_t rbd_list_children(rbd_image_t image, char *pools, size_t *pools_len, char *images, size_t *images_len); -/* cooperative locking */ /** - * in params: - * @param lockers_and_cookies: array of char* which will be filled in - * @param max_entries: the size of the lockers_and_cookies array - * out params: - * @param exclusive: non-zero if the lock is an exclusive one. Only - * meaningfull if there are a non-zero number of lockers. - * @param lockers_and_cookies: alternating the address of the locker with - * the locker's cookie. - * @param max_entries: the number of lockers -- double for the number of - * spaces required. + * @defgroup librbd_h_locking Advisory Locking + * + * An rbd image may be locking exclusively, or shared, to facilitate + * e.g. live migration where the image may be open in two places at once. + * These locks are intended to guard against more than one client + * writing to an image without coordination. They don't need to + * be used for snapshots, since snapshots are read-only. + * + * Currently locks only guard against locks being acquired. + * They do not prevent anything else. + * + * A locker is identified by the internal rados client id of the + * holder and a user-defined cookie. This (client id, cookie) pair + * must be unique for each locker. + * + * A shared lock also has a user-defined tag associated with it. Each + * additional shared lock must specify the same tag or lock + * acquisition will fail. This can be used by e.g. groups of hosts + * using a clustered filesystem on top of an rbd image to make sure + * they're accessing the correct image. + * + * @{ + */ +/** + * List clients that have locked the image and information about the lock. + * + * The number of bytes required in each buffer is put in the + * corresponding size out parameter. If any of the provided buffers + * are too short, -ERANGE is returned after these sizes are filled in. + * + * @param exclusive where to store whether the lock is exclusive (1) or shared (0) + * @param tag where to store the tag associated with the image + * @param tag_len number of bytes in tag buffer + * @param clients buffer in which locker clients are stored, separated by '\0' + * @param clients_len number of bytes in the clients buffer + * @param cookies buffer in which locker cookies are stored, separated by '\0' + * @param cookies_len number of bytes in the cookies buffer + * @param addrs buffer in which locker addresses are stored, separated by '\0' + * @param addrs_len number of bytes in the clients buffer + * @returns number of lockers on success, negative error code on failure + * @returns -ERANGE if any of the buffers are too short + */ +ssize_t rbd_list_lockers(rbd_image_t image, int *exclusive, + char *tag, size_t *tag_len, + char *clients, size_t *clients_len, + char *cookies, size_t *cookies_len, + char *addrs, size_t *addrs_len); + +/** + * Take an exclusive lock on the image. + * + * @param image the image to lock + * @param cookie user-defined identifier for this instance of the lock + * @returns 0 on success, negative error code on failure + * @returns -EBUSY if the lock is already held by another (client, cookie) pair + * @returns -EEXIST if the lock is already held by the same (client, cookie) pair */ -int rbd_list_lockers(rbd_image_t image, int *exclusive, - char **lockers_and_cookies, int *max_entries); int rbd_lock_exclusive(rbd_image_t image, const char *cookie); -int rbd_lock_shared(rbd_image_t image, const char *cookie); + +/** + * Take a shared lock on the image. + * + * Other clients may also take a shared lock, as lock as they use the + * same tag. + * + * @param image the image to lock + * @param cookie user-defined identifier for this instance of the lock + * @param tag user-defined identifier for this shared use of the lock + * @returns 0 on success, negative error code on failure + * @returns -EBUSY if the lock is already held by another (client, cookie) pair + * @returns -EEXIST if the lock is already held by the same (client, cookie) pair + */ +int rbd_lock_shared(rbd_image_t image, const char *cookie, const char *tag); + +/** + * Release a shared or exclusive lock on the image. + * + * @param image the image to unlock + * @param cookie user-defined identifier for the instance of the lock + * @returns 0 on success, negative error code on failure + * @returns -ENOENT if the lock is not held by the specified (client, cookie) pair + */ int rbd_unlock(rbd_image_t image, const char *cookie); -int rbd_break_lock(rbd_image_t image, const char *locker, const char *cookie); +/** + * Release a shared or exclusive lock that was taken by the specified client. + * + * @param image the image to unlock + * @param client the entity holding the lock (as given by rbd_list_lockers()) + * @param cookie user-defined identifier for the instance of the lock to break + * @returns 0 on success, negative error code on failure + * @returns -ENOENT if the lock is not held by the specified (client, cookie) pair + */ +int rbd_break_lock(rbd_image_t image, const char *client, const char *cookie); + +/** @} locking */ /* I/O */ typedef void *rbd_completion_t; diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 1608e693850cf..ec22113a86c72 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -39,6 +39,12 @@ namespace librbd { std::string name; } snap_info_t; + typedef struct { + std::string client; + std::string cookie; + std::string address; + } locker_t; + typedef rbd_image_info_t image_info_t; class ProgressContext @@ -110,13 +116,13 @@ public: */ int list_children(std::set > *children); - /* cooperative locking */ - int list_locks(std::set > &locks, - bool &exclusive); + /* advisory locking (see librbd.h for details) */ + int list_lockers(std::list *lockers, + bool *exclusive, std::string *tag); int lock_exclusive(const std::string& cookie); - int lock_shared(const std::string& cookie); + int lock_shared(const std::string& cookie, const std::string& tag); int unlock(const std::string& cookie); - int break_lock(const std::string& other_locker, const std::string& cookie); + int break_lock(const std::string& client, const std::string& cookie); /* snapshots */ int snap_list(std::vector& snaps); diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 766ec097c082f..8538f88d53575 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -40,8 +40,12 @@ namespace librbd { std::map snaps_by_name; uint64_t snap_id; bool snap_exists; // false if our snap_id was deleted - std::set > locks; + + std::map lockers; bool exclusive_locked; + std::string lock_tag; + std::string name; std::string snap_name; IoCtx data_ctx, md_ctx; diff --git a/src/librbd/cls_rbd_client.cc b/src/librbd/cls_rbd_client.cc index a259ac95f98f6..c5441ab0b6c58 100644 --- a/src/librbd/cls_rbd_client.cc +++ b/src/librbd/cls_rbd_client.cc @@ -1,8 +1,10 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "cls/lock/cls_lock_client.h" #include "include/buffer.h" #include "include/encoding.h" +#include "include/rbd_types.h" #include "cls_rbd_client.h" @@ -22,6 +24,7 @@ namespace librbd { ::encode(snap, bl); op.exec("rbd", "get_size", bl); op.exec("rbd", "get_object_prefix", empty); + bufferlist outbl; int r = ioctx->operate(oid, &op, &outbl); @@ -46,8 +49,10 @@ namespace librbd { int get_mutable_metadata(librados::IoCtx *ioctx, const std::string &oid, uint64_t *size, uint64_t *features, uint64_t *incompatible_features, - std::set > *lockers, + map *lockers, bool *exclusive_lock, + string *lock_tag, ::SnapContext *snapc, parent_info *parent) { @@ -68,8 +73,8 @@ namespace librbd { op.exec("rbd", "get_size", sizebl); op.exec("rbd", "get_features", featuresbl); op.exec("rbd", "get_snapcontext", empty); - op.exec("rbd", "list_locks", empty); op.exec("rbd", "get_parent", parentbl); + rados::cls::lock::get_lock_info_start(&op, RBD_LOCK_NAME); bufferlist outbl; int r = ioctx->operate(oid, &op, &outbl); @@ -87,14 +92,20 @@ namespace librbd { ::decode(*incompatible_features, iter); // get_snapcontext ::decode(*snapc, iter); - // list_locks - ::decode(*lockers, iter); - ::decode(*exclusive_lock, iter); // get_parent ::decode(parent->spec.pool_id, iter); ::decode(parent->spec.image_id, iter); ::decode(parent->spec.snap_id, iter); ::decode(parent->overlap, iter); + + // get_lock_info + ClsLockType lock_type; + r = rados::cls::lock::get_lock_info_finish(&iter, lockers, &lock_type, + lock_tag); + if (r < 0) + return r; + + *exclusive_lock = (lock_type == LOCK_EXCLUSIVE); } catch (const buffer::error &err) { return -EBADMSG; } diff --git a/src/librbd/cls_rbd_client.h b/src/librbd/cls_rbd_client.h index 3fb595adece3c..9036a7b255f7f 100644 --- a/src/librbd/cls_rbd_client.h +++ b/src/librbd/cls_rbd_client.h @@ -4,6 +4,7 @@ #ifndef CEPH_LIBRBD_CLS_RBD_CLIENT_H #define CEPH_LIBRBD_CLS_RBD_CLIENT_H +#include "cls/lock/cls_lock_types.h" #include "common/snap_types.h" #include "include/rados.h" #include "include/rados/librados.hpp" @@ -21,8 +22,10 @@ namespace librbd { int get_mutable_metadata(librados::IoCtx *ioctx, const std::string &oid, uint64_t *size, uint64_t *features, uint64_t *incompatible_features, - std::set >* lockers, + map *lockers, bool *exclusive_lock, + std::string *lock_tag, ::SnapContext *snapc, parent_info *parent); diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 5d5261da24180..7b84bfcc35a8e 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -6,6 +6,8 @@ #include "common/ceph_context.h" #include "common/dout.h" #include "common/errno.h" +#include "cls/lock/cls_lock_client.h" +#include "include/stringify.h" #include "librbd/AioCompletion.h" #include "librbd/AioRequest.h" @@ -262,6 +264,8 @@ namespace librbd { if (ictx) { assert(ictx->md_lock.is_locked()); ictx->refresh_lock.Lock(); + ldout(ictx->cct, 20) << "notify_change refresh_seq = " << ictx->refresh_seq + << " last_refresh = " << ictx->last_refresh << dendl; ++ictx->refresh_seq; ictx->refresh_lock.Unlock(); } @@ -1486,6 +1490,7 @@ reprotect_and_return_err: Mutex::Locker l(ictx->snap_lock); { Mutex::Locker l2(ictx->parent_lock); + ictx->lockers.clear(); if (ictx->old_format) { r = read_header(ictx->md_ctx, ictx->header_oid, &ictx->header, NULL); if (r < 0) { @@ -1495,9 +1500,20 @@ reprotect_and_return_err: r = cls_client::old_snapshot_list(&ictx->md_ctx, ictx->header_oid, &snap_names, &snap_sizes, &new_snapc); if (r < 0) { - lderr(cct) << "Error listing snapshots: " << cpp_strerror(r) << dendl; + lderr(cct) << "Error listing snapshots: " << cpp_strerror(r) + << dendl; + return r; + } + ClsLockType lock_type; + r = rados::cls::lock::get_lock_info(&ictx->md_ctx, ictx->header_oid, + RBD_LOCK_NAME, &ictx->lockers, + &lock_type, &ictx->lock_tag); + if (r < 0) { + lderr(cct) << "Error getting lock info: " << cpp_strerror(r) + << dendl; return r; } + ictx->exclusive_locked = (lock_type == LOCK_EXCLUSIVE); ictx->order = ictx->header.options.order; ictx->size = ictx->header.image_size; ictx->object_prefix = ictx->header.block_name; @@ -1507,8 +1523,9 @@ reprotect_and_return_err: r = cls_client::get_mutable_metadata(&ictx->md_ctx, ictx->header_oid, &ictx->size, &ictx->features, &incompatible_features, - &ictx->locks, + &ictx->lockers, &ictx->exclusive_locked, + &ictx->lock_tag, &new_snapc, &ictx->parent_md); if (r < 0) { @@ -1754,7 +1771,7 @@ reprotect_and_return_err: { ldout(ictx->cct, 20) << "open_image: ictx = " << ictx << " name = '" << ictx->name - << " id = '" << ictx->id + << "' id = '" << ictx->id << "' snap_name = '" << ictx->snap_name << "'" << dendl; int r = ictx->init(); @@ -1917,9 +1934,10 @@ reprotect_and_return_err: return r; } - int list_locks(ImageCtx *ictx, - set > &locks, - bool &exclusive) + int list_lockers(ImageCtx *ictx, + std::list *lockers, + bool *exclusive, + string *tag) { ldout(ictx->cct, 20) << "list_locks on image " << ictx << dendl; @@ -1928,40 +1946,94 @@ reprotect_and_return_err: return r; Mutex::Locker locker(ictx->md_lock); - locks = ictx->locks; - exclusive = ictx->exclusive_locked; + if (exclusive) + *exclusive = ictx->exclusive_locked; + if (tag) + *tag = ictx->lock_tag; + if (lockers) { + lockers->clear(); + map::const_iterator it; + for (it = ictx->lockers.begin(); it != ictx->lockers.end(); ++it) { + locker_t locker; + locker.client = stringify(it->first.locker); + locker.cookie = it->first.cookie; + locker.address = stringify(it->second.addr); + lockers->push_back(locker); + } + } + return 0; } - int lock_exclusive(ImageCtx *ictx, const string& cookie) + int lock(ImageCtx *ictx, bool exclusive, const string& cookie, + const string& tag) { + ldout(ictx->cct, 20) << "lock image " << ictx << " exclusive=" << exclusive + << " cookie='" << cookie << "' tag='" << tag << "'" + << dendl; + + int r = ictx_check(ictx); + if (r < 0) + return r; + /** * If we wanted we could do something more intelligent, like local * checks that we think we will succeed. But for now, let's not * duplicate that code. */ - return cls::lock::lock(&ictx->md_ctx, ictx->header_oid, - RBD_LOCK_NAME, LOCK_EXCLUSIVE, - cookie, "", "", utime_t(), 0) - cookie); - } - - int lock_shared(ImageCtx *ictx, const string& cookie) - { - return cls_client::lock_image_shared(&ictx->md_ctx, - ictx->header_oid, cookie); + Mutex::Locker locker(ictx->md_lock); + r = rados::cls::lock::lock(&ictx->md_ctx, ictx->header_oid, RBD_LOCK_NAME, + exclusive ? LOCK_EXCLUSIVE : LOCK_SHARED, + cookie, tag, "", utime_t(), 0); + if (r < 0) + return r; + notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx); + return 0; } int unlock(ImageCtx *ictx, const string& cookie) { - return cls_client::unlock_image(&ictx->md_ctx, ictx->header_oid, cookie); + ldout(ictx->cct, 20) << "unlock image " << ictx + << " cookie='" << cookie << "'" << dendl; + + + int r = ictx_check(ictx); + if (r < 0) + return r; + + Mutex::Locker locker(ictx->md_lock); + r = rados::cls::lock::unlock(&ictx->md_ctx, ictx->header_oid, + RBD_LOCK_NAME, cookie); + if (r < 0) + return r; + notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx); + return 0; } - int break_lock(ImageCtx *ictx, const string& lock_holder, + int break_lock(ImageCtx *ictx, const string& client, const string& cookie) { - return cls_client::break_lock(&ictx->md_ctx, ictx->header_oid, - lock_holder, cookie); + ldout(ictx->cct, 20) << "break_lock image " << ictx << " client='" << client + << "' cookie='" << cookie << "'" << dendl; + + int r = ictx_check(ictx); + if (r < 0) + return r; + + entity_name_t lock_client; + if (!lock_client.parse(client)) { + lderr(ictx->cct) << "Unable to parse client '" << client + << "'" << dendl; + return -EINVAL; + } + Mutex::Locker locker(ictx->md_lock); + r = rados::cls::lock::break_lock(&ictx->md_ctx, ictx->header_oid, + RBD_LOCK_NAME, cookie, lock_client); + if (r < 0) + return r; + notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx); + return 0; } void rbd_ctx_cb(completion_t cb, void *arg) diff --git a/src/librbd/internal.h b/src/librbd/internal.h index 2421e0bc6624a..2e1bd8675a791 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -121,13 +121,16 @@ namespace librbd { int flatten(ImageCtx *ictx, ProgressContext &prog_ctx); /* cooperative locking */ - int list_locks(ImageCtx *ictx, - std::set > &locks, - bool &exclusive); - int lock_exclusive(ImageCtx *ictx, const std::string& cookie); - int lock_shared(ImageCtx *ictx, const std::string& cookie); + int list_lockers(ImageCtx *ictx, + std::list *locks, + bool *exclusive, + std::string *tag); + int lock(ImageCtx *ictx, bool exclusive, const std::string& cookie, + const std::string& tag); + int lock_shared(ImageCtx *ictx, const std::string& cookie, + const std::string& tag); int unlock(ImageCtx *ictx, const std::string& cookie); - int break_lock(ImageCtx *ictx, const std::string& lock_holder, + int break_lock(ImageCtx *ictx, const std::string& client, const std::string& cookie); void trim_image(ImageCtx *ictx, uint64_t newsize, ProgressContext& prog_ctx); diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index fab565b4c9fa1..dab0cafed1747 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -272,32 +272,35 @@ namespace librbd { return librbd::list_children(ictx, *children); } - int Image::list_locks(set > &locks, - bool &exclusive) + int Image::list_lockers(std::list *lockers, + bool *exclusive, string *tag) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::list_locks(ictx, locks, exclusive); + return librbd::list_lockers(ictx, lockers, exclusive, tag); } int Image::lock_exclusive(const string& cookie) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::lock_exclusive(ictx, cookie); + return librbd::lock(ictx, true, cookie, ""); } - int Image::lock_shared(const string& cookie) + + int Image::lock_shared(const string& cookie, const std::string& tag) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::lock_shared(ictx, cookie); + return librbd::lock(ictx, false, cookie, tag); } + int Image::unlock(const string& cookie) { ImageCtx *ictx = (ImageCtx *)ctx; return librbd::unlock(ictx, cookie); } - int Image::break_lock(const string& other_locker, const string& cookie) + + int Image::break_lock(const string& client, const string& cookie) { ImageCtx *ictx = (ImageCtx *)ctx; - return librbd::break_lock(ictx, other_locker, cookie); + return librbd::break_lock(ictx, client, cookie); } int Image::snap_create(const char *snap_name) @@ -784,63 +787,86 @@ extern "C" ssize_t rbd_list_children(rbd_image_t image, char *pools, return image_set.size(); } -extern "C" int rbd_list_lockers(rbd_image_t image, int *exclusive, - char **lockers_and_cookies, int *max_entries) +extern "C" ssize_t rbd_list_lockers(rbd_image_t image, int *exclusive, + char *tag, size_t *tag_len, + char *clients, size_t *clients_len, + char *cookies, size_t *cookies_len, + char *addrs, size_t *addrs_len) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - set > locks; + std::list lockers; bool exclusive_bool; + string tag_str; - if (*max_entries <= 0) { - return -ERANGE; - } - - int r = list_locks(ictx, locks, exclusive_bool); - if (r < 0) { + int r = list_lockers(ictx, &lockers, &exclusive_bool, &tag_str); + if (r < 0) return r; - } + + ldout(ictx->cct, 20) << "list_lockers r = " << r << " lockers.size() = " << lockers.size() << dendl; + *exclusive = (int)exclusive_bool; - bool fits = locks.size() * 2 <= (size_t)*max_entries; - *max_entries = locks.size() * 2; - if (!fits) { + size_t clients_total = 0; + size_t cookies_total = 0; + size_t addrs_total = 0; + for (list::const_iterator it = lockers.begin(); + it != lockers.end(); ++it) { + clients_total += it->client.length() + 1; + cookies_total += it->cookie.length() + 1; + addrs_total += it->address.length() + 1; + } + + bool too_short = ((clients_total > *clients_len) || + (cookies_total > *cookies_len) || + (addrs_total > *addrs_len) || + (tag_str.length() + 1 > *tag_len)); + *clients_len = clients_total; + *cookies_len = cookies_total; + *addrs_len = addrs_total; + *tag_len = tag_str.length() + 1; + if (too_short) return -ERANGE; - } - set >::iterator p; - int i = 0; - for (p = locks.begin(); - p != locks.end(); - ++p) { - lockers_and_cookies[i] = strdup(p->first.c_str()); - lockers_and_cookies[i+1] = strdup(p->second.c_str()); - i+=2; + strcpy(tag, tag_str.c_str()); + char *clients_p = clients; + char *cookies_p = cookies; + char *addrs_p = addrs; + for (list::const_iterator it = lockers.begin(); + it != lockers.end(); ++it) { + strcpy(clients_p, it->client.c_str()); + clients_p += it->client.length() + 1; + strcpy(cookies_p, it->cookie.c_str()); + cookies_p += it->cookie.length() + 1; + strcpy(addrs_p, it->address.c_str()); + addrs_p += it->address.length() + 1; } - return 0; + + return lockers.size(); } extern "C" int rbd_lock_exclusive(rbd_image_t image, const char *cookie) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::lock_exclusive(ictx, cookie); + return librbd::lock(ictx, true, cookie ? cookie : "", ""); } -extern "C" int rbd_lock_shared(rbd_image_t image, const char *cookie) +extern "C" int rbd_lock_shared(rbd_image_t image, const char *cookie, + const char *tag) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::lock_shared(ictx, cookie); + return librbd::lock(ictx, false, cookie ? cookie : "", tag ? tag : ""); } extern "C" int rbd_unlock(rbd_image_t image, const char *cookie) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::unlock(ictx, cookie); + return librbd::unlock(ictx, cookie ? cookie : ""); } -extern "C" int rbd_break_lock(rbd_image_t image, const char *locker, - const char *cookie) +extern "C" int rbd_break_lock(rbd_image_t image, const char *client, + const char *cookie) { librbd::ImageCtx *ictx = (librbd::ImageCtx *)image; - return librbd::break_lock(ictx, locker, cookie); + return librbd::break_lock(ictx, client, cookie ? cookie : ""); } /* I/O */ diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py index 2bb911dafb33d..93e5e17941e3c 100644 --- a/src/pybind/rbd.py +++ b/src/pybind/rbd.py @@ -646,6 +646,107 @@ written." % (self.name, ret, length)) images = c_images.raw[:images_size.value - 1].split('\0') return zip(pools, images) + def list_lockers(self): + """ + List clients that have locked the image and information + about the lock. + + :returns: dict - contains keys tag, exclusive, and lockers + tag - the tag associated with the lock + (every additional locker must use the same tag) + exclusive - boolean indicating whether the lock is + exclusive or shared + lockers - a list of (client, cookie, address) tuples + """ + clients_size = c_size_t(512) + cookies_size = c_size_t(512) + addrs_size = c_size_t(512) + tag_size = c_size_t(512) + exclusive = c_int(0) + + while True: + c_clients = create_string_buffer(clients_size.value) + c_cookies = create_string_buffer(cookies_size.value) + c_addrs = create_string_buffer(addrs_size.value) + c_tag = create_string_buffer(tag_size.value) + ret = self.librbd.rbd_list_lockers(self.image, + byref(exclusive), + byref(c_tag), + byref(tag_size), + byref(c_clients), + byref(clients_size), + byref(c_cookies), + byref(cookies_size), + byref(c_addrs), + byref(addrs_size)) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error listing images') + if ret == 0: + return [] + clients = c_clients.raw[:clients_size.value - 1].split('\0') + cookies = c_cookies.raw[:cookies_size.value - 1].split('\0') + addrs = c_addrs.raw[:addrs_size.value - 1].split('\0') + return { + 'tag' : c_tag.value, + 'exclusive' : exclusive.value == 1, + 'lockers' : zip(clients, cookies, addrs), + } + + def lock_exclusive(self, cookie): + """ + Take an exclusive lock on the image. + + :raises: :class:`ImageBusy` if a different client or cookie locked it + :class:`ImageExists` if the same client and cookie locked it + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_lock_exclusive(self.image, c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error acquiring exclusive lock on image') + + def lock_shared(self, cookie, tag): + """ + Take a shared lock on the image. The tag must match + that of the existing lockers, if any. + + :raises: :class:`ImageBusy` if a different client or cookie locked it + :class:`ImageExists` if the same client and cookie locked it + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + if not isinstance(tag, str): + raise TypeError('tag must be a string') + ret = self.librbd.rbd_lock_shared(self.image, c_char_p(cookie), + c_char_p(tag)) + if ret < 0: + raise make_ex(ret, 'error acquiring shared lock on image') + + def unlock(self, cookie): + """ + Release a lock on the image that was locked by this rados client. + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_unlock(self.image, c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error unlocking image') + + def break_lock(self, client, cookie): + """ + Release a lock held by another rados client. + """ + if not isinstance(client, str): + raise TypeError('client must be a string') + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_break_lock(self.image, c_char_p(client), + c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error unlocking image') + class SnapIterator(object): """ diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index d898d8e7374dc..42a317298ba14 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -395,6 +395,41 @@ class TestImage(object): read = self.image.read(0, 256) eq(read, data) self.image.remove_snap('snap1') + + def test_lock_unlock(self): + assert_raises(ImageNotFound, self.image.unlock, '') + self.image.lock_exclusive('') + assert_raises(ImageExists, self.image.lock_exclusive, '') + assert_raises(ImageBusy, self.image.lock_exclusive, 'test') + assert_raises(ImageExists, self.image.lock_shared, '', '') + assert_raises(ImageBusy, self.image.lock_shared, 'foo', '') + self.image.unlock('') + + def test_list_lockers(self): + eq([], self.image.list_lockers()) + self.image.lock_exclusive('test') + lockers = self.image.list_lockers() + eq(1, len(lockers['lockers'])) + _, cookie, _ = lockers['lockers'][0] + eq(cookie, 'test') + eq('', lockers['tag']) + assert lockers['exclusive'] + self.image.unlock('test') + eq([], self.image.list_lockers()) + + num_shared = 10 + for i in xrange(num_shared): + self.image.lock_shared(str(i), 'tag') + lockers = self.image.list_lockers() + eq('tag', lockers['tag']) + assert not lockers['exclusive'] + eq(num_shared, len(lockers['lockers'])) + cookies = sorted(map(lambda x: x[1], lockers['lockers'])) + for i in xrange(num_shared): + eq(str(i), cookies[i]) + self.image.unlock(str(i)) + eq([], self.image.list_lockers()) + class TestClone(object): diff --git a/src/test/test_librbd.cc b/src/test/test_librbd.cc index 023d39c1055e0..b74bb233215c1 100644 --- a/src/test/test_librbd.cc +++ b/src/test/test_librbd.cc @@ -33,6 +33,7 @@ #include "rados-api/test.h" #include "common/errno.h" +#include "include/stringify.h" using namespace std; @@ -1229,3 +1230,74 @@ TEST(LibRBD, ListChildren) ASSERT_EQ(0, rados_pool_delete(cluster, pool_name1.c_str())); ASSERT_EQ(0, destroy_one_pool(pool_name2, &cluster)); } + +TEST(LibRBD, LockingPP) +{ + librados::Rados rados; + librados::IoCtx ioctx; + string pool_name = get_temp_pool_name(); + + ASSERT_EQ("", create_one_pool_pp(pool_name, rados)); + ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx)); + + { + librbd::RBD rbd; + librbd::Image image; + int order = 0; + const char *name = "testimg"; + uint64_t size = 2 << 20; + std::string cookie1 = "foo"; + std::string cookie2 = "bar"; + + ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order)); + ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL)); + + // no lockers initially + std::list lockers; + std::string tag; + bool exclusive; + ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag)); + ASSERT_EQ(0u, lockers.size()); + ASSERT_EQ("", tag); + + // exclusive lock is exclusive + ASSERT_EQ(0, image.lock_exclusive(cookie1)); + ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie1)); + ASSERT_EQ(-EBUSY, image.lock_exclusive("")); + ASSERT_EQ(-EEXIST, image.lock_shared(cookie1, "")); + ASSERT_EQ(-EBUSY, image.lock_shared(cookie1, "test")); + ASSERT_EQ(-EBUSY, image.lock_shared("", "test")); + ASSERT_EQ(-EBUSY, image.lock_shared("", "")); + + // list exclusive + ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag)); + ASSERT_TRUE(exclusive); + ASSERT_EQ("", tag); + ASSERT_EQ(1u, lockers.size()); + ASSERT_EQ(cookie1, lockers.front().cookie); + + // unlock + ASSERT_EQ(-ENOENT, image.unlock("")); + ASSERT_EQ(-ENOENT, image.unlock(cookie2)); + ASSERT_EQ(0, image.unlock(cookie1)); + ASSERT_EQ(-ENOENT, image.unlock(cookie1)); + ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag)); + ASSERT_EQ(0u, lockers.size()); + + ASSERT_EQ(0, image.lock_shared(cookie1, "")); + ASSERT_EQ(-EEXIST, image.lock_shared(cookie1, "")); + ASSERT_EQ(0, image.lock_shared(cookie2, "")); + ASSERT_EQ(-EEXIST, image.lock_shared(cookie2, "")); + ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie1)); + ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie2)); + ASSERT_EQ(-EBUSY, image.lock_exclusive("")); + ASSERT_EQ(-EBUSY, image.lock_exclusive("test")); + + // list shared + ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag)); + ASSERT_EQ(2u, lockers.size()); + } + + ioctx.close(); + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados)); +} -- 2.39.5