From fe0c78587aa24ada6f990d3c454e7e13f14fc68e Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 10 Jul 2015 17:10:25 -0700 Subject: [PATCH] rgw: add new class to create resource request RGWRESTReadResource holds the request input and output. Previously we were using stack variables that couldn't be used in the async path. Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_http_client.cc | 136 +++++++++++++++++++++++++++++++------ src/rgw/rgw_http_client.h | 18 +++-- src/rgw/rgw_op.cc | 1 + src/rgw/rgw_rados.cc | 1 + src/rgw/rgw_rados.h | 2 +- src/rgw/rgw_rest_client.cc | 75 ++++++++++++++++++++ src/rgw/rgw_rest_client.h | 13 +++- src/rgw/rgw_rest_conn.cc | 67 +++++++++++++----- src/rgw/rgw_rest_conn.h | 97 +++++++++++++++++++++++++- src/rgw/rgw_rest_s3.h | 1 + src/rgw/rgw_sync.cc | 17 +++-- 11 files changed, 374 insertions(+), 54 deletions(-) diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc index 923dd8ca16431..615548a4c63dc 100644 --- a/src/rgw/rgw_http_client.cc +++ b/src/rgw/rgw_http_client.cc @@ -7,6 +7,8 @@ #include "rgw_common.h" #include "rgw_http_client.h" +#include "rgw_http_errors.h" +#include "common/RefCountedObj.h" #define dout_subsys ceph_subsys_rgw @@ -117,23 +119,60 @@ int RGWHTTPClient::process(const char *method, const char *url) return ret; } -struct rgw_http_req_data { +struct rgw_http_req_data : public RefCountedObject { CURL *easy_handle; curl_slist *h; uint64_t id; + int ret; + atomic_t done; + RGWHTTPClient *client; + RGWHTTPManager *mgr; + + Mutex lock; + Cond cond; - rgw_http_req_data() : easy_handle(NULL), h(NULL), id(-1) {} + rgw_http_req_data() : easy_handle(NULL), h(NULL), id(-1), ret(0), client(NULL), + mgr(NULL), lock("rgw_http_req_data::lock") {} ~rgw_http_req_data() { + if (mgr) { + mgr->remove_request(id); + } + } + + int wait() { + Mutex::Locker l(lock); + cond.Wait(lock); + return ret; + } + + void finish(int r) { + Mutex::Locker l(lock); + ret = r; + cond.Signal(); + done.set(1); + if (easy_handle) curl_easy_cleanup(easy_handle); if (h) curl_slist_free_all(h); + + easy_handle = NULL; + h = NULL; + } + + bool is_done() { + return done.read() != 0; } + }; -int RGWHTTPClient::init_request(const char *method, const char *url, rgw_http_req_data *req_data) +int RGWHTTPClient::init_request(const char *method, const char *url, rgw_http_req_data *_req_data) { + assert(!req_data); + _req_data->get(); + req_data = _req_data; + CURL *easy_handle; char error_buf[CURL_ERROR_SIZE]; @@ -171,6 +210,23 @@ int RGWHTTPClient::init_request(const char *method, const char *url, rgw_http_re return 0; } +int RGWHTTPClient::wait() +{ + if (!req_data->is_done()) { + return req_data->wait(); + } + + return req_data->ret; +} + +RGWHTTPClient::~RGWHTTPClient() +{ + if (req_data) { + req_data->put(); + } +} + + #if HAVE_CURL_MULTI_WAIT static int do_curl_wait(CephContext *cct, CURLM *handle, int signal_fd) @@ -256,31 +312,56 @@ void RGWHTTPManager::register_request(rgw_http_req_data *req_data) { RWLock::WLocker rl(reqs_lock); req_data->id = num_reqs; -ldout(cct, 0) << __FILE__ << ":" << __LINE__ << " req_data->id=" << req_data->id << ", easy_handle=" << req_data->easy_handle << dendl; reqs[num_reqs] = req_data; num_reqs++; + ldout(cct, 20) << __func__ << " mgr=" << this << " req_data->id=" << req_data->id << ", easy_handle=" << req_data->easy_handle << dendl; } -void RGWHTTPManager::unregister_request(rgw_http_req_data *req_data) +void RGWHTTPManager::complete_request(rgw_http_req_data *req_data) { RWLock::WLocker rl(reqs_lock); + _complete_request(req_data); +} + +void RGWHTTPManager::_complete_request(rgw_http_req_data *req_data) +{ map::iterator iter = reqs.find(req_data->id); + if (iter != reqs.end()) { + reqs.erase(iter); + } + complete_reqs[req_data->id] = req_data; +dout(0) << __FILE__ << ":" << __LINE__ << ": _complete_request() id=" << req_data->id << " complete_reqs.size()=" << complete_reqs.size() << dendl; +} + +void RGWHTTPManager::remove_request(uint64_t id) +{ + RWLock::WLocker rl(reqs_lock); + map::iterator iter = reqs.find(id); + if (iter != reqs.end()) { + reqs.erase(iter); + } + iter = complete_reqs.find(id); if (iter == reqs.end()) { - return; + complete_reqs.erase(iter); } - reqs.erase(iter); +dout(0) << __FILE__ << ":" << __LINE__ << ": remove_request() id=" << id << dendl; } -void RGWHTTPManager::finish_request(rgw_http_req_data *req_data) +void RGWHTTPManager::finish_request(rgw_http_req_data *req_data, int ret) { - unregister_request(req_data); + req_data->finish(ret); + complete_request(req_data); +} - delete req_data; +void RGWHTTPManager::_finish_request(rgw_http_req_data *req_data, int ret) +{ + req_data->finish(ret); + _complete_request(req_data); } int RGWHTTPManager::link_request(rgw_http_req_data *req_data) { -ldout(cct, 0) << __FILE__ << ":" << __LINE__ << " req_data->id=" << req_data->id << ", easy_handle=" << req_data->easy_handle << dendl; + ldout(cct, 20) << __func__ << " req_data=" << req_data << " req_data->id=" << req_data->id << ", easy_handle=" << req_data->easy_handle << dendl; CURLMcode mstatus = curl_multi_add_handle((CURLM *)multi_handle, req_data->easy_handle); if (mstatus) { dout(0) << "ERROR: failed on curl_multi_add_handle, status=" << mstatus << dendl; @@ -307,8 +388,7 @@ void RGWHTTPManager::link_pending_requests() int r = link_request(req_data); if (r < 0) { ldout(cct, 0) << "ERROR: failed to link http request" << dendl; -#warning FIXME: need to send back error on request - delete req_data; + _finish_request(req_data, r); } max_threaded_req = iter->first + 1; } @@ -320,23 +400,25 @@ int RGWHTTPManager::add_request(RGWHTTPClient *client, const char *method, const int ret = client->init_request(method, url, req_data); if (ret < 0) { + delete req_data; return ret; } + req_data->mgr = this; + req_data->client = client; + register_request(req_data); if (!is_threaded) { ret = link_request(req_data); - } else { - ret = signal_thread(); + return ret; } + ret = signal_thread(); if (ret < 0) { - delete req_data; -#warning should drop reference here - return ret; + finish_request(req_data, ret); } - return 0; + return ret; } int RGWHTTPManager::process_requests(bool wait_for_data, bool *done) @@ -370,7 +452,13 @@ int RGWHTTPManager::process_requests(bool wait_for_data, bool *done) CURL *e = msg->easy_handle; rgw_http_req_data *req_data; curl_easy_getinfo(e, CURLINFO_PRIVATE, (void **)&req_data); - finish_request(req_data); + + long http_status; + curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, (void **)&http_status); + + int status = rgw_http_error_to_errno(http_status); + finish_request(req_data, status); + req_data->put(); switch (msg->data.result) { case CURLE_OK: break; @@ -466,7 +554,13 @@ void *RGWHTTPManager::reqs_thread_entry() CURL *e = msg->easy_handle; rgw_http_req_data *req_data; curl_easy_getinfo(e, CURLINFO_PRIVATE, (void **)&req_data); - finish_request(req_data); + + long http_status; + curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, (void **)&http_status); + + int status = rgw_http_error_to_errno(http_status); + finish_request(req_data, status); + req_data->put(); switch (msg->data.result) { case CURLE_OK: break; diff --git a/src/rgw/rgw_http_client.h b/src/rgw/rgw_http_client.h index 9b36dc722d81e..95c5d21bea598 100644 --- a/src/rgw/rgw_http_client.h +++ b/src/rgw/rgw_http_client.h @@ -18,14 +18,17 @@ class RGWHTTPClient bufferlist::iterator send_iter; size_t send_len; bool has_send_len; + + rgw_http_req_data *req_data; + protected: CephContext *cct; list > headers; int init_request(const char *method, const char *url, rgw_http_req_data *req_data); public: - virtual ~RGWHTTPClient() {} - explicit RGWHTTPClient(CephContext *_cct): send_len (0), has_send_len(false), cct(_cct) {} + explicit RGWHTTPClient(CephContext *_cct): send_len (0), has_send_len(false), req_data(NULL), cct(_cct) {} + virtual ~RGWHTTPClient(); void append_header(const string& name, const string& val) { headers.push_back(pair(name, val)); @@ -42,6 +45,9 @@ public: int process(const char *method, const char *url); int process(const char *url) { return process("GET", url); } + + int wait(); + rgw_http_req_data *get_req_data() { return req_data; } }; class RGWHTTPManager { @@ -52,13 +58,16 @@ class RGWHTTPManager { RWLock reqs_lock; map reqs; + map complete_reqs; int64_t num_reqs; int64_t max_threaded_req; int thread_pipe[2]; void register_request(rgw_http_req_data *req_data); - void unregister_request(rgw_http_req_data *req_data); - void finish_request(rgw_http_req_data *req_data); + void complete_request(rgw_http_req_data *req_data); + void _complete_request(rgw_http_req_data *req_data); + void finish_request(rgw_http_req_data *req_data, int r); + void _finish_request(rgw_http_req_data *req_data, int r); int link_request(rgw_http_req_data *req_data); void link_pending_requests(); @@ -85,6 +94,7 @@ public: void stop(); int add_request(RGWHTTPClient *client, const char *method, const char *url); + void remove_request(uint64_t id); /* only for non threaded case */ int process_requests(bool wait_for_data, bool *done); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 26560524c581e..9e07f32da75f9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -25,6 +25,7 @@ #include "rgw_multi_del.h" #include "rgw_cors.h" #include "rgw_cors_s3.h" +#include "rgw_rest_conn.h" #include "rgw_client_io.h" diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index c592ab262a94c..b36d4cd1d7b95 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -19,6 +19,7 @@ #include "rgw_acl_s3.h" /* for dumping s3policy in debug log */ #include "rgw_metadata.h" #include "rgw_bucket.h" +#include "rgw_rest_conn.h" #include "cls/rgw/cls_rgw_ops.h" #include "cls/rgw/cls_rgw_types.h" diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 06af3af91e7c7..1aaa54ba4030f 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -16,13 +16,13 @@ #include "cls/timeindex/cls_timeindex_types.h" #include "rgw_log.h" #include "rgw_metadata.h" -#include "rgw_rest_conn.h" class RGWWatcher; class SafeTimer; class ACLOwner; class RGWGC; class RGWObjectExpirer; +class RGWRESTConn; /* flags for put_obj_meta() */ #define PUT_OBJ_CREATE 0x01 diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 0a2b814e79ec4..0c1ed25740040 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -694,3 +694,78 @@ int RGWRESTStreamReadRequest::send_data(void *ptr, size_t len) return 0; } +class StreamIntoBufferlist : public RGWGetDataCB { + bufferlist& bl; +public: + StreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {} + int handle_data(bufferlist& inbl, off_t bl_ofs, off_t bl_len) { + bl.claim_append(inbl); + return bl_len; + } +}; + +#if 0 +struct RGWRESTRequestParams { + string resource; + list > *params; + map *headers; + + RGWRESTRequestParams() : params(NULL), headers(NULL) {} +}; + + +class RGWRESTBufferReadRequest { + RGWRESTConn *conn; + RGWRESTRequestParams *params; + bufferlist bl; + StreamIntoBufferlist cb; + RGWRESTStreamReadRequest req; + + list > http_params; + map http_headers; +public: + RGWRESTBufferReadRequest(RGWRESTConn *_conn, + RGWRESTRequestParams *_params, + RGWHTTPManager *mgr = NULL) : conn(_conn), params(_params), cb(bl) { + if (params->params) { + list >::iterator iter = params->params->begin(); + for (; iter != params->params->end(); ++iter) { + http_params.push_back(*iter); + } + } + http_params.push_back(pair(RGW_SYS_PARAM_PREFIX "region", conn->get_region())); + + if (params->headers) { + for (map::iterator iter = params->headers->begin(); + iter != params->headers->end(); ++iter) { + http_headers[iter->first] = iter->second; + } + } + } + + + int get(); +}; + +int RGWRESTBufferReadRequest::get() +{ + string url; + int ret = conn->get_url(url); + if (ret < 0) + return ret; + + req.init(cct, url, &cb, NULL, &http_params); + + ret = req.get_resource(key, http_headers, resource, mgr); + if (ret < 0) { + ldout(cct, 0) << __func__ << ": get() resource=" << resource << " returned ret=" << ret << dendl; + return ret; + } + + string etag; + map attrs; + return req.complete(etag, NULL, attrs); +} + + +#endif diff --git a/src/rgw/rgw_rest_client.h b/src/rgw/rgw_rest_client.h index a88f0f1e72312..4967f9bab512a 100644 --- a/src/rgw/rgw_rest_client.h +++ b/src/rgw/rgw_rest_client.h @@ -31,13 +31,20 @@ protected: int sign_request(RGWAccessKey& key, RGWEnv& env, req_info& info); public: - RGWRESTSimpleRequest(CephContext *_cct, string& _url, list > *_headers, + RGWRESTSimpleRequest(CephContext *_cct, const string& _url, list > *_headers, list > *_params) : RGWHTTPClient(_cct), http_status(0), status(0), url(_url), send_iter(NULL), max_response(0) { + set_headers(_headers); + set_params(_params); + } + + void set_headers(list > *_headers) { if (_headers) headers = *_headers; + } + void set_params(list > *_params) { if (_params) params = *_params; } @@ -64,7 +71,7 @@ public: int add_output_data(bufferlist& bl); int send_data(void *ptr, size_t len); - RGWRESTStreamWriteRequest(CephContext *_cct, string& _url, list > *_headers, + RGWRESTStreamWriteRequest(CephContext *_cct, const string& _url, list > *_headers, list > *_params) : RGWRESTSimpleRequest(_cct, _url, _headers, _params), lock("RGWRESTStreamWriteRequest"), cb(NULL), http_manager(_cct) {} ~RGWRESTStreamWriteRequest(); @@ -87,7 +94,7 @@ public: int send_data(void *ptr, size_t len); int receive_data(void *ptr, size_t len); - RGWRESTStreamReadRequest(CephContext *_cct, string& _url, RGWGetDataCB *_cb, list > *_headers, + RGWRESTStreamReadRequest(CephContext *_cct, const string& _url, RGWGetDataCB *_cb, list > *_headers, list > *_params) : RGWRESTSimpleRequest(_cct, _url, _headers, _params), lock("RGWRESTStreamReadRequest"), cb(_cb), chunk_ofs(0), ofs(0), http_manager(_cct) {} diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc index ed435995691b1..815789c9965d4 100644 --- a/src/rgw/rgw_rest_conn.cc +++ b/src/rgw/rgw_rest_conn.cc @@ -30,6 +30,20 @@ int RGWRESTConn::get_url(string& endpoint) return 0; } +string RGWRESTConn::get_url() +{ + string endpoint; + if (endpoints.empty()) { + ldout(cct, 0) << "WARNING: endpoints not configured for upstream zone" << dendl; /* we'll catch this later */ + return endpoint; + } + + int i = counter.inc(); + endpoint = endpoints[i % endpoints.size()]; + + return endpoint; +} + int RGWRESTConn::forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl) { string url; @@ -122,16 +136,6 @@ int RGWRESTConn::complete_request(RGWRESTStreamReadRequest *req, string& etag, t return ret; } -class StreamIntoBufferlist : public RGWGetDataCB { - bufferlist& bl; -public: - StreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {} - int handle_data(bufferlist& inbl, off_t bl_ofs, off_t bl_len) { - bl.claim_append(inbl); - return bl_len; - } -}; - int RGWRESTConn::get_resource(const string& resource, list > *extra_params, map *extra_headers, @@ -154,7 +158,7 @@ int RGWRESTConn::get_resource(const string& resource, params.push_back(pair(RGW_SYS_PARAM_PREFIX "region", region)); - StreamIntoBufferlist cb(bl); + RGWStreamIntoBufferlist cb(bl); RGWRESTStreamReadRequest req(cct, url, &cb, NULL, ¶ms); @@ -177,11 +181,12 @@ int RGWRESTConn::get_resource(const string& resource, return req.complete(etag, NULL, attrs); } -int RGWRESTConn::send_get_resource(const string& resource, const rgw_http_param_pair *pp, - bufferlist &bl, RGWHTTPManager *mgr) -{ - list > params; - +RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn, + const string& _resource, + const rgw_http_param_pair *pp, + list > *extra_headers, + RGWHTTPManager *_mgr) : cct(_conn->get_ctx()), conn(_conn), resource(_resource), cb(bl), + mgr(_mgr), req(cct, conn->get_url(), &cb, NULL, NULL) { while (pp && pp->key) { string k = pp->key; string v = (pp->val ? pp->val : ""); @@ -189,8 +194,36 @@ int RGWRESTConn::send_get_resource(const string& resource, const rgw_http_param_ ++pp; } - int ret = get_resource(resource, ¶ms, NULL, bl, mgr); + params.push_back(pair(RGW_SYS_PARAM_PREFIX "region", conn->get_region())); + + if (extra_headers) { + for (list >::iterator iter = extra_headers->begin(); + iter != extra_headers->end(); ++iter) { + headers[iter->first] = iter->second; + } + } + + req.set_params(¶ms); +} + +int RGWRESTReadResource::read() +{ + int ret = req.get_resource(conn->get_key(), headers, resource, mgr); + if (ret < 0) { + ldout(cct, 0) << __func__ << ": get_resource() resource=" << resource << " returned ret=" << ret << dendl; + return ret; + } + + string etag; + map attrs; + return req.complete(etag, NULL, attrs); +} + +int RGWRESTReadResource::aio_read() +{ + int ret = req.get_resource(conn->get_key(), headers, resource, mgr); if (ret < 0) { + ldout(cct, 0) << __func__ << ": get_resource() resource=" << resource << " returned ret=" << ret << dendl; return ret; } diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index 997cca970e5c4..6f7ce32398f73 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -4,6 +4,7 @@ #ifndef CEPH_RGW_REST_CONN_H #define CEPH_RGW_REST_CONN_H +#include "rgw_rados.h" #include "rgw_rest_client.h" #include "common/ceph_json.h" @@ -46,6 +47,17 @@ public: RGWRESTConn(CephContext *_cct, RGWRados *store, list& endpoints); int get_url(string& endpoint); + string get_url(); + const string& get_region() { + return region; + } + RGWAccessKey& get_key() { + return key; + } + + CephContext *get_ctx() { + return cct; + } /* sync request */ int forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl); @@ -63,8 +75,6 @@ public: map* extra_headers, bufferlist& bl, RGWHTTPManager *mgr = NULL); - int send_get_resource(const string& resource, const rgw_http_param_pair *pp, - bufferlist &bl, RGWHTTPManager *mgr); template int get_json_resource(const string& resource, list > *params, T& t); template @@ -104,4 +114,87 @@ int RGWRESTConn::get_json_resource(const string& resource, const rgw_http_param return get_json_resource(resource, ¶ms, t); } +class RGWStreamIntoBufferlist : public RGWGetDataCB { + bufferlist& bl; +public: + RGWStreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {} + int handle_data(bufferlist& inbl, off_t bl_ofs, off_t bl_len) { + bl.claim_append(inbl); + return bl_len; + } +}; + +class RGWRESTReadResource { + CephContext *cct; + RGWRESTConn *conn; + string resource; + list > params; + map headers; + bufferlist bl; + RGWStreamIntoBufferlist cb; + + RGWHTTPManager *mgr; + RGWRESTStreamReadRequest req; + +public: + RGWRESTReadResource(RGWRESTConn *_conn, + const string& _resource, + const rgw_http_param_pair *pp, + list > *extra_headers, + RGWHTTPManager *_mgr); + + template + int decode_resource(T *dest); + + int read(); + + int aio_read(); + + template + int wait(T *dest); + + template + int get(T *dest); +}; + + +template +int RGWRESTReadResource::decode_resource(T *dest) +{ + int ret = parse_decode_json(cct, *dest, bl); + if (ret < 0) { + return ret; + } + return 0; +} + +template +int RGWRESTReadResource::get(T *dest) +{ + int ret = read(); + if (ret < 0) { + return ret; + } + + ret = decode_resource(dest); + if (ret < 0) { + return ret; + } + return 0; +} + +template +int RGWRESTReadResource::wait(T *dest) +{ + int ret = req.wait(); + if (ret < 0) { + return ret; + } + + ret = decode_resource(dest); + if (ret < 0) { + return ret; + } + return 0; +} #endif diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index ada556290f771..88f674b79b1af 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -10,6 +10,7 @@ #include "rgw_acl_s3.h" #include "rgw_policy_s3.h" #include "rgw_keystone.h" +#include "rgw_rest_conn.h" #define RGW_AUTH_GRACE_MINS 15 diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 1fcac5c1a6969..4da713b7ca301 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -5,6 +5,7 @@ #include "rgw_rados.h" #include "rgw_sync.h" #include "rgw_metadata.h" +#include "rgw_rest_conn.h" #define dout_subsys ceph_subsys_rgw @@ -199,17 +200,21 @@ int RGWRemoteMetaLog::clone_shard(int shard_id, const string& marker, string *ne { marker_key, marker.c_str() }, { NULL, NULL } }; - bufferlist bl; - int ret = conn->send_get_resource("/admin/log", pairs, bl, &http_manager); + RGWRESTReadResource http_op(conn, "/admin/log", pairs, NULL, &http_manager); + + rgw_mdlog_shard_data data; + + int ret = http_op.aio_read(); if (ret < 0) { ldout(store->ctx(), 0) << "ERROR: failed to fetch mdlog data" << dendl; return ret; } -#warning removeme -sleep(7); - - rgw_mdlog_shard_data data; + ret = http_op.wait(&data); + if (ret < 0) { + ldout(store->ctx(), 0) << "ERROR: failed to wait for op, ret=" << ret << dendl; + return ret; + } ldout(store->ctx(), 20) << "remote mdlog, shard_id=" << shard_id << " num of shard entries: " << data.entries.size() << dendl; -- 2.39.5