rgw: add RGWReshard class

Signed-off-by: Orit Wasserman <owasserm@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index f693913bf99ea11f187d65d65bb289523d0f6196..9c0c626ba0538efcb422cbc5692a6d1e02f4a3c6 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1731,3 +1731,6 @@ OPTION(internal_safe_to_start_threads, OPT_BOOL, false)
 OPTION(debug_deliberately_leak_memory, OPT_BOOL, false)
 
 OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers
+
+/* resharding tunables */
+OPTION(rgw_reshard_max_jobs, OPT_INT, 1024)

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index c0e0c907b758b440dca86280bcb1e87dc05148e8..2e3733d626d962c7f975b667c1f7bf5eaa869840 100644 (file)
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -81,6 +81,7 @@ set(rgw_a_srcs
   rgw_period_pusher.cc
   rgw_realm_reloader.cc
   rgw_realm_watcher.cc
+  rgw_reshard.cc
   rgw_coroutine.cc
   rgw_cr_rados.cc
   rgw_object_expirer_core.cc

diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc
new file mode 100644 (file)
index 0000000..bcc2f33
--- /dev/null
+++ b/src/rgw/rgw_reshard.cc
@@ -0,0 +1,185 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw_rados.h"
+#include "rgw_reshard.h"
+#include "cls/rgw/cls_rgw_client.h"
+#include "cls/lock/cls_lock_client.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+const string reshard_oid = "reshard";
+const string reshard_lock_name = "reshard_process";
+
+RGWReshard::RGWReshard(CephContext *_cct, RGWRados* _store):cct(_cct), store(_store)
+{
+    max_jobs = cct->_conf->rgw_reshard_max_jobs;
+}
+
+int RGWReshard::get_io_ctx(librados::IoCtx& io_ctx)
+{
+  string reshard_pool = store->get_zone_params().reshard_pool.name;
+  librados::Rados *rad = store->get_rados_handle();
+  int r = rad->ioctx_create(reshard_pool.c_str(), io_ctx);
+  if (r == -ENOENT) {
+    r = store->create_pool(store->get_zone_params().reshard_pool);
+    if (r < 0)
+      return r;
+
+    // retry
+    r = rad->ioctx_create(reshard_pool.c_str(), io_ctx);
+  }
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWReshard::add(cls_rgw_reshard_entry& entry)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_exclusive(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshard::add failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  librados::ObjectWriteOperation op;
+  cls_rgw_reshard_add(op, entry);
+
+  ret = io_ctx.operate(reshard_oid, &op);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}
+
+
+int RGWReshard::list(string& marker, uint32_t max, std::list<cls_rgw_reshard_entry>& entries, bool& is_truncated)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_shared(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshard::list failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  ret =  cls_rgw_reshard_list(io_ctx, reshard_oid, marker, max, entries, is_truncated);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}
+
+int RGWReshard::get_head(cls_rgw_reshard_entry& entry)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_shared(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshardLog::add failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  ret = cls_rgw_reshard_get_head(io_ctx, reshard_oid, entry);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}
+
+int RGWReshard::get(cls_rgw_reshard_entry& entry)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_shared(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshardLog::get failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  ret = cls_rgw_reshard_get(io_ctx, reshard_oid, entry);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}
+
+int RGWReshard::remove(cls_rgw_reshard_entry& entry)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_exclusive(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshardLog::remove failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  librados::ObjectWriteOperation op;
+  cls_rgw_reshard_remove(op, entry);
+
+  ret =  io_ctx.operate(reshard_oid, &op);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}
+
+int RGWReshard::remove(cls_rgw_reshard_entry& entry)
+{
+  rados::cls::lock::Lock l(reshard_lock_name);
+  librados::IoCtx io_ctx;
+
+  int ret = get_io_ctx(io_ctx);
+  if (ret < 0)
+    return ret;
+
+  ret = l.lock_exclusive(&io_ctx, reshard_oid);
+  if (ret == -EBUSY) {
+    dout(0) << "RGWReshardLog::add failed to acquire lock on " << reshard_oid << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  librados::ObjectWriteOperation op;
+  cls_rgw_reshard_remove(op);
+
+  ret =  io_ctx.operate(reshard_oid, &op);
+
+  l.unlock(&io_ctx, reshard_oid);
+  return ret;
+}

diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h
new file mode 100644 (file)
index 0000000..91987d4
--- /dev/null
+++ b/src/rgw/rgw_reshard.h
@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RGW_RESHARD_H
+#define RGW_RESHARD_H
+
+#include <vector>
+#include "include/rados/librados.hpp"
+#include "cls/rgw/cls_rgw_types.h"
+
+class CephContext;
+class RGWRados;
+
+class RGWReshard {
+    CephContext *cct;
+    RGWRados *store;
+    string lock_name;
+    int max_jobs;
+
+    int get_io_ctx(librados::IoCtx& io_ctx);
+  public:
+    RGWReshard(CephContext* cct, RGWRados* _store);
+    int add(cls_rgw_reshard_entry& entry);
+    int get_head(cls_rgw_reshard_entry& entry);
+    int get(cls_rgw_reshard_entry& entry);
+    int remove(cls_rgw_reshard_entry& entry);
+    int list(string& marker, uint32_t max, list<cls_rgw_reshard_entry>& entries, bool& is_truncated);
+};
+
+#endif