From b14f85ab6b878ca981b5840683b0483372b661ac Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 11 Aug 2015 00:41:39 -0700 Subject: [PATCH] rgw: mdlog sync init initialize mdlog sync state, uses the implicit yield scheme (from boost) Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_admin.cc | 19 +++++++++++++ src/rgw/rgw_coroutine.cc | 2 +- src/rgw/rgw_coroutine.h | 39 ++++++++++++------------- src/rgw/rgw_sync.cc | 61 +++++++++++++++++++++++++++++++++++++--- src/rgw/rgw_sync.h | 2 ++ 5 files changed, 97 insertions(+), 26 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 2c3e50dc112fc..14dac62d601f8 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -292,6 +292,7 @@ enum { OPT_MDLOG_TRIM, OPT_MDLOG_FETCH, OPT_MDLOG_SYNC_STATUS, + OPT_MDLOG_SYNC_INIT, OPT_BILOG_LIST, OPT_BILOG_TRIM, OPT_DATALOG_LIST, @@ -567,6 +568,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_ (strcmp(prev_cmd, "sync") == 0)) { if (strcmp(cmd, "status") == 0) return OPT_MDLOG_SYNC_STATUS; + if (strcmp(cmd, "init") == 0) + return OPT_MDLOG_SYNC_INIT; } else if (strcmp(prev_cmd, "bilog") == 0) { if (strcmp(cmd, "list") == 0) return OPT_BILOG_LIST; @@ -3354,6 +3357,22 @@ next: } + if (opt_cmd == OPT_MDLOG_SYNC_INIT) { + RGWMetadataSync sync(store); + + int ret = sync.init(); + if (ret < 0) { + cerr << "ERROR: sync.init() returned ret=" << ret << std::endl; + return -ret; + } + + ret = sync.init_sync_status(); + if (ret < 0) { + cerr << "ERROR: sync.get_sync_status() returned ret=" << ret << std::endl; + return -ret; + } + } + if (opt_cmd == OPT_BILOG_LIST) { if (bucket_name.empty()) { cerr << "ERROR: bucket not specified" << std::endl; diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc index 4ebb724ab7b28..822b0b5113ed7 100644 --- a/src/rgw/rgw_coroutine.cc +++ b/src/rgw/rgw_coroutine.cc @@ -257,7 +257,7 @@ int RGWSimpleCoroutine::operate() yield return state_all_complete(); } - return 0; + return set_state(RGWCoroutine_Done); } int RGWSimpleCoroutine::state_init() diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h index 0b23ded951096..e1ef7570d396c 100644 --- a/src/rgw/rgw_coroutine.h +++ b/src/rgw/rgw_coroutine.h @@ -54,15 +54,29 @@ struct RGWCoroutinesEnv { RGWCoroutinesEnv() : manager(NULL), stacks(NULL), stack(NULL) {} }; +enum RGWCoroutineState { + RGWCoroutine_Error = -2, + RGWCoroutine_Done = -1, + RGWCoroutine_Run = 0, +}; + class RGWCoroutine : public RefCountedObject, public boost::asio::coroutine { friend class RGWCoroutinesStack; + protected: + CephContext *cct; + RGWCoroutinesEnv *env; bool blocked; int retcode; + int state; stringstream error_stream; + int set_state(int s, int ret = 0) { + state = s; + return ret; + } void set_blocked(bool flag) { blocked = flag; } int block(int ret) { set_blocked(true); @@ -78,13 +92,13 @@ protected: void spawn(RGWCoroutine *op); public: - RGWCoroutine() : env(NULL), blocked(false), retcode(0) {} + RGWCoroutine(CephContext *_cct) : cct(_cct), env(NULL), blocked(false), retcode(0), state(RGWCoroutine_Run) {} virtual ~RGWCoroutine() {} virtual int operate() = 0; - virtual bool is_done() = 0; - virtual bool is_error() = 0; + bool is_done() { return (state == RGWCoroutine_Done || state == RGWCoroutine_Error); } + bool is_error() { return (state == RGWCoroutine_Error); } stringstream& log_error() { return error_stream; } string error_str() { @@ -181,19 +195,7 @@ public: } }; -enum RGWCoroutineState { - RGWCoroutine_Run = 0, - RGWCoroutine_Done = 1, - RGWCoroutine_Error = 2, -}; - class RGWSimpleCoroutine : public RGWCoroutine { - int state; - - int set_state(int s, int ret = 0) { - state = s; - return ret; - } int operate(); int state_init(); @@ -201,19 +203,14 @@ class RGWSimpleCoroutine : public RGWCoroutine { int state_request_complete(); int state_all_complete(); -protected: - CephContext *cct; - public: - RGWSimpleCoroutine(CephContext *_cct) : state(RGWCoroutine_Run), cct(_cct) {} + RGWSimpleCoroutine(CephContext *_cct) : RGWCoroutine(_cct) {} virtual int init() { return 0; } virtual int send_request() = 0; virtual int request_complete() = 0; virtual int finish() { return 0; } - bool is_done() { return (state == RGWCoroutine_Done || state == RGWCoroutine_Error); } - bool is_error() { return (state == RGWCoroutine_Error); } }; #endif diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 242b3b0aec83a..3b55403d5053a 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -13,6 +13,9 @@ #include "cls/lock/cls_lock_client.h" +#include +#include + #define dout_subsys ceph_subsys_rgw @@ -646,8 +649,7 @@ class RGWSimpleRadosUnlockCR : public RGWSimpleCoroutine { public: RGWSimpleRadosUnlockCR(RGWAsyncRadosProcessor *_async_rados, RGWRados *_store, rgw_bucket& _pool, const string& _oid, const string& _lock_name, - const string& _cookie, - uint32_t _duration) : RGWSimpleCoroutine(_store->ctx()), + const string& _cookie) : RGWSimpleCoroutine(_store->ctx()), async_rados(_async_rados), store(_store), lock_name(_lock_name), @@ -673,6 +675,51 @@ public: } }; +class RGWInitSyncStatusCoroutine : public RGWCoroutine { + RGWAsyncRadosProcessor *async_rados; + RGWRados *store; + RGWObjectCtx& obj_ctx; + + string lock_name; + string cookie; + +public: + RGWInitSyncStatusCoroutine(RGWAsyncRadosProcessor *_async_rados, RGWRados *_store, + RGWObjectCtx& _obj_ctx) : RGWCoroutine(_store->ctx()), async_rados(_async_rados), store(_store), + obj_ctx(_obj_ctx) { + lock_name = "sync_lock"; + +#define COOKIE_LEN 16 + char buf[COOKIE_LEN + 1]; + + gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1); + string cookie = buf; + } + + int operate() { + reenter(this) { + yield { + uint32_t lock_duration = 30; + call(new RGWSimpleRadosLockCR(async_rados, store, store->get_zone_params().log_pool, "mdlog.state.global", + lock_name, cookie, lock_duration)); + } + yield { + call(new RGWSimpleRadosUnlockCR(async_rados, store, store->get_zone_params().log_pool, "mdlog.state.global", + lock_name, cookie)); + } + yield { + rgw_sync_marker sync_marker; + call(new RGWSimpleRadosWriteCR(async_rados, store, store->get_zone_params().log_pool, + "mdlog.state.global", sync_marker)); + } + yield { + return set_state(RGWCoroutine_Done); + } + } + return 0; + } +}; + class RGWReadSyncStatusCoroutine : public RGWSimpleRadosReadCR { RGWAsyncRadosProcessor *async_rados; RGWRados *store; @@ -731,7 +778,7 @@ class RGWMetaSyncCoroutine : public RGWCoroutine { return ret; } public: - RGWMetaSyncCoroutine(RGWRados *_store, RGWHTTPManager *_mgr, int _id) : RGWCoroutine(), store(_store), + RGWMetaSyncCoroutine(RGWRados *_store, RGWHTTPManager *_mgr, int _id) : RGWCoroutine(_store->ctx()), store(_store), mdlog(store->meta_mgr->get_log()), http_manager(_mgr), shard_id(_id), @@ -824,7 +871,7 @@ class RGWCloneMetaLogCoroutine : public RGWCoroutine { } public: RGWCloneMetaLogCoroutine(RGWRados *_store, RGWHTTPManager *_mgr, - int _id, const string& _marker) : RGWCoroutine(), store(_store), + int _id, const string& _marker) : RGWCoroutine(_store->ctx()), store(_store), mdlog(store->meta_mgr->get_log()), http_manager(_mgr), shard_id(_id), marker(_marker), truncated(false), max_entries(CLONE_MAX_ENTRIES), @@ -886,6 +933,12 @@ int RGWRemoteMetaLog::get_sync_status(RGWMetaSyncGlobalStatus *sync_status) return run(new RGWReadSyncStatusCoroutine(async_rados, store, obj_ctx, sync_status)); } +int RGWRemoteMetaLog::init_sync_status() +{ + RGWObjectCtx obj_ctx(store, NULL); + return run(new RGWInitSyncStatusCoroutine(async_rados, store, obj_ctx)); +} + int RGWRemoteMetaLog::get_shard_sync_marker(int shard_id, rgw_sync_marker *shard_status) { int ret = status_manager.read_shard_status(shard_id); diff --git a/src/rgw/rgw_sync.h b/src/rgw/rgw_sync.h index e242ef850ce04..74101a1f70bba 100644 --- a/src/rgw/rgw_sync.h +++ b/src/rgw/rgw_sync.h @@ -161,6 +161,7 @@ public: int clone_shards(); int fetch(); int get_sync_status(RGWMetaSyncGlobalStatus *sync_status); + int init_sync_status(); int get_shard_sync_marker(int shard_id, rgw_sync_marker *shard_status); }; @@ -177,6 +178,7 @@ public: int init(); int get_sync_status(RGWMetaSyncGlobalStatus *sync_status) { return master_log.get_sync_status(sync_status); } + int init_sync_status() { return master_log.init_sync_status(); } int get_shard_sync_marker(int shard_id, rgw_sync_marker *shard_status) { return master_log.get_shard_sync_marker(shard_id, shard_status); } -- 2.39.5