From a34f6af72689d64594bfd2c41f58a584c0d96971 Mon Sep 17 00:00:00 2001 From: Orit Wasserman Date: Wed, 8 Mar 2017 17:56:39 +0200 Subject: [PATCH] rgw: add RGWReshard class Signed-off-by: Orit Wasserman --- src/common/config_opts.h | 3 + src/rgw/CMakeLists.txt | 1 + src/rgw/rgw_reshard.cc | 185 +++++++++++++++++++++++++++++++++++++++ src/rgw/rgw_reshard.h | 30 +++++++ 4 files changed, 219 insertions(+) create mode 100644 src/rgw/rgw_reshard.cc create mode 100644 src/rgw/rgw_reshard.h diff --git a/src/common/config_opts.h b/src/common/config_opts.h index f693913bf99..9c0c626ba05 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1731,3 +1731,6 @@ OPTION(internal_safe_to_start_threads, OPT_BOOL, false) OPTION(debug_deliberately_leak_memory, OPT_BOOL, false) OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers + +/* resharding tunables */ +OPTION(rgw_reshard_max_jobs, OPT_INT, 1024) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index c0e0c907b75..2e3733d626d 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -81,6 +81,7 @@ set(rgw_a_srcs rgw_period_pusher.cc rgw_realm_reloader.cc rgw_realm_watcher.cc + rgw_reshard.cc rgw_coroutine.cc rgw_cr_rados.cc rgw_object_expirer_core.cc diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc new file mode 100644 index 00000000000..bcc2f33795a --- /dev/null +++ b/src/rgw/rgw_reshard.cc @@ -0,0 +1,185 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "rgw_rados.h" +#include "rgw_reshard.h" +#include "cls/rgw/cls_rgw_client.h" +#include "cls/lock/cls_lock_client.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +const string reshard_oid = "reshard"; +const string reshard_lock_name = "reshard_process"; + +RGWReshard::RGWReshard(CephContext *_cct, RGWRados* _store):cct(_cct), store(_store) +{ + max_jobs = cct->_conf->rgw_reshard_max_jobs; +} + +int RGWReshard::get_io_ctx(librados::IoCtx& io_ctx) +{ + string reshard_pool = store->get_zone_params().reshard_pool.name; + librados::Rados *rad = store->get_rados_handle(); + int r = rad->ioctx_create(reshard_pool.c_str(), io_ctx); + if (r == -ENOENT) { + r = store->create_pool(store->get_zone_params().reshard_pool); + if (r < 0) + return r; + + // retry + r = rad->ioctx_create(reshard_pool.c_str(), io_ctx); + } + if (r < 0) + return r; + + return 0; +} + +int RGWReshard::add(cls_rgw_reshard_entry& entry) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_exclusive(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshard::add failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + librados::ObjectWriteOperation op; + cls_rgw_reshard_add(op, entry); + + ret = io_ctx.operate(reshard_oid, &op); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} + + +int RGWReshard::list(string& marker, uint32_t max, std::list& entries, bool& is_truncated) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_shared(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshard::list failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + ret = cls_rgw_reshard_list(io_ctx, reshard_oid, marker, max, entries, is_truncated); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} + +int RGWReshard::get_head(cls_rgw_reshard_entry& entry) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_shared(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshardLog::add failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + ret = cls_rgw_reshard_get_head(io_ctx, reshard_oid, entry); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} + +int RGWReshard::get(cls_rgw_reshard_entry& entry) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_shared(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshardLog::get failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + ret = cls_rgw_reshard_get(io_ctx, reshard_oid, entry); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} + +int RGWReshard::remove(cls_rgw_reshard_entry& entry) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_exclusive(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshardLog::remove failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + librados::ObjectWriteOperation op; + cls_rgw_reshard_remove(op, entry); + + ret = io_ctx.operate(reshard_oid, &op); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} + +int RGWReshard::remove(cls_rgw_reshard_entry& entry) +{ + rados::cls::lock::Lock l(reshard_lock_name); + librados::IoCtx io_ctx; + + int ret = get_io_ctx(io_ctx); + if (ret < 0) + return ret; + + ret = l.lock_exclusive(&io_ctx, reshard_oid); + if (ret == -EBUSY) { + dout(0) << "RGWReshardLog::add failed to acquire lock on " << reshard_oid << dendl; + return 0; + } + if (ret < 0) + return ret; + + librados::ObjectWriteOperation op; + cls_rgw_reshard_remove(op); + + ret = io_ctx.operate(reshard_oid, &op); + + l.unlock(&io_ctx, reshard_oid); + return ret; +} diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h new file mode 100644 index 00000000000..91987d4d7a4 --- /dev/null +++ b/src/rgw/rgw_reshard.h @@ -0,0 +1,30 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RGW_RESHARD_H +#define RGW_RESHARD_H + +#include +#include "include/rados/librados.hpp" +#include "cls/rgw/cls_rgw_types.h" + +class CephContext; +class RGWRados; + +class RGWReshard { + CephContext *cct; + RGWRados *store; + string lock_name; + int max_jobs; + + int get_io_ctx(librados::IoCtx& io_ctx); + public: + RGWReshard(CephContext* cct, RGWRados* _store); + int add(cls_rgw_reshard_entry& entry); + int get_head(cls_rgw_reshard_entry& entry); + int get(cls_rgw_reshard_entry& entry); + int remove(cls_rgw_reshard_entry& entry); + int list(string& marker, uint32_t max, list& entries, bool& is_truncated); +}; + +#endif -- 2.39.5