From 2dcafc722a39f91fe9de20775528df3d519b2690 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 4 Aug 2016 09:17:54 -0700 Subject: [PATCH] rgw: propagate attrs, mtime, size of remote object Use new rgwx-stat http param that allows getting only object's meta. Use that when calling stat_remote_object(). Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_cr_rados.cc | 6 +++--- src/rgw/rgw_cr_rados.h | 32 ++++++++++++++++++++++---------- src/rgw/rgw_op.h | 2 ++ src/rgw/rgw_rados.cc | 17 +++++++++++------ src/rgw/rgw_rados.h | 3 ++- src/rgw/rgw_rest.cc | 2 ++ src/rgw/rgw_rest_client.cc | 32 ++++++++++++++++++++++---------- src/rgw/rgw_rest_client.h | 2 +- src/rgw/rgw_rest_conn.cc | 18 +++++++++++------- src/rgw/rgw_rest_conn.h | 4 ++-- src/rgw/rgw_rest_s3.cc | 10 ++++++++++ 11 files changed, 88 insertions(+), 40 deletions(-) diff --git a/src/rgw/rgw_cr_rados.cc b/src/rgw/rgw_cr_rados.cc index 5e54cc4ef2534..8c8ccc7f05a8f 100644 --- a/src/rgw/rgw_cr_rados.cc +++ b/src/rgw/rgw_cr_rados.cc @@ -534,7 +534,6 @@ int RGWAsyncStatRemoteObj::_send_request() snprintf(buf, sizeof(buf), ".%lld", (long long)store->instance_id()); string client_id = store->zone_id() + buf; string op_id = store->unique_id(store->get_new_req_id()); - map attrs; rgw_obj src_obj(bucket_info.bucket, key.name); src_obj.set_instance(key.instance); @@ -548,13 +547,14 @@ int RGWAsyncStatRemoteObj::_send_request() source_zone, src_obj, bucket_info, /* source */ - nullptr, /* real_time* src_mtime, */ + pmtime, /* real_time* src_mtime, */ + psize, /* uint64_t * */ nullptr, /* const real_time* mod_ptr, */ nullptr, /* const real_time* unmod_ptr, */ true, /* high precision time */ nullptr, /* const char *if_match, */ nullptr, /* const char *if_nomatch, */ - attrs, + pattrs, nullptr, nullptr, /* string *ptag, */ nullptr); /* string *petag, */ diff --git a/src/rgw/rgw_cr_rados.h b/src/rgw/rgw_cr_rados.h index 4265a4192c833..ee02dd8a33abb 100644 --- a/src/rgw/rgw_cr_rados.h +++ b/src/rgw/rgw_cr_rados.h @@ -2,6 +2,7 @@ #define CEPH_RGW_CR_RADOS_H #include "rgw_coroutine.h" +#include "rgw_rados.h" #include "common/WorkQueue.h" #include "common/Throttle.h" @@ -796,9 +797,10 @@ class RGWAsyncStatRemoteObj : public RGWAsyncRadosRequest { RGWBucketInfo bucket_info; rgw_obj_key key; - uint64_t versioned_epoch; - real_time src_mtime; + ceph::real_time *pmtime; + uint64_t *psize; + map *pattrs; protected: int _send_request(); @@ -807,11 +809,15 @@ public: const string& _source_zone, RGWBucketInfo& _bucket_info, const rgw_obj_key& _key, - uint64_t _versioned_epoch) : RGWAsyncRadosRequest(caller, cn), store(_store), + ceph::real_time *_pmtime, + uint64_t *_psize, + map *_pattrs) : RGWAsyncRadosRequest(caller, cn), store(_store), source_zone(_source_zone), bucket_info(_bucket_info), key(_key), - versioned_epoch(_versioned_epoch) {} + pmtime(_pmtime), + psize(_psize), + pattrs(_pattrs) {} }; class RGWStatRemoteObjCR : public RGWSimpleCoroutine { @@ -823,7 +829,10 @@ class RGWStatRemoteObjCR : public RGWSimpleCoroutine { RGWBucketInfo bucket_info; rgw_obj_key key; - uint64_t versioned_epoch; + + ceph::real_time *pmtime; + uint64_t *psize; + map *pattrs; RGWAsyncStatRemoteObj *req; @@ -832,13 +841,16 @@ public: const string& _source_zone, RGWBucketInfo& _bucket_info, const rgw_obj_key& _key, - uint64_t _versioned_epoch, - bool _if_newer) : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()), + ceph::real_time *_pmtime, + uint64_t *_psize, + map *_pattrs) : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()), async_rados(_async_rados), store(_store), source_zone(_source_zone), bucket_info(_bucket_info), key(_key), - versioned_epoch(_versioned_epoch), + pmtime(_pmtime), + psize(_psize), + pattrs(_pattrs), req(NULL) {} @@ -854,8 +866,8 @@ public: } int send_request() { - req = new RGWAsyncStatRemoteObj(this, stack->create_completion_notifier(), store, source_zone, bucket_info, - key, versioned_epoch); + req = new RGWAsyncStatRemoteObj(this, stack->create_completion_notifier(), store, source_zone, + bucket_info, key, pmtime, psize, pattrs); async_rados->queue(req); return 0; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index b52242331624c..375f3b01d34d2 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -133,6 +133,7 @@ protected: utime_t gc_invalidate_time; bool is_slo; string lo_etag; + bool rgwx_stat; /* extended rgw stat operation */ int init_common(); public: @@ -155,6 +156,7 @@ public: range_parsed = false; skip_manifest = false; is_slo = false; + rgwx_stat = false; } bool prefetch_data(); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index acc5b73a6e27a..dc03f28a37b85 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -6771,12 +6771,13 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx, rgw_obj& src_obj, RGWBucketInfo& src_bucket_info, real_time *src_mtime, + uint64_t *psize, const real_time *mod_ptr, const real_time *unmod_ptr, bool high_precision_time, const char *if_match, const char *if_nomatch, - map& attrs, + map *pattrs, string *version_id, string *ptag, string *petag) @@ -6823,12 +6824,13 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx, int ret = conn->get_obj(user_id, info, src_obj, pmod, unmod_ptr, dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver, - true, false, &cb, &in_stream_req); + true /* prepend_meta */, true /* GET */, true /* rgwx-stat */, + &cb, &in_stream_req); if (ret < 0) { return ret; } - ret = conn->complete_request(in_stream_req, etag, &set_mtime, req_headers); + ret = conn->complete_request(in_stream_req, etag, &set_mtime, psize, req_headers); if (ret < 0) { return ret; } @@ -6858,7 +6860,9 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx, } } - attrs = src_attrs; + if (pattrs) { + *pattrs = src_attrs; + } return 0; } @@ -6980,12 +6984,13 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, ret = conn->get_obj(user_id, info, src_obj, pmod, unmod_ptr, dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver, - true, true, &cb, &in_stream_req); + true /* prepend_meta */, true /* GET */, false /* rgwx-stat */, + &cb, &in_stream_req); if (ret < 0) { goto set_err_state; } - ret = conn->complete_request(in_stream_req, etag, &set_mtime, req_headers); + ret = conn->complete_request(in_stream_req, etag, &set_mtime, nullptr, req_headers); if (ret < 0) { goto set_err_state; } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index e13d36ae40bed..fb956f64fdcca 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2566,12 +2566,13 @@ public: rgw_obj& src_obj, RGWBucketInfo& src_bucket_info, real_time *src_mtime, + uint64_t *psize, const real_time *mod_ptr, const real_time *unmod_ptr, bool high_precision_time, const char *if_match, const char *if_nomatch, - map& attrs, + map *pattrs, string *version_id, string *ptag, string *petag); diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index bd357b3b643c5..06da365b6842d 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -816,6 +816,8 @@ int RGWGetObj_ObjStore::get_params() if (s->system_request) { mod_zone_id = s->info.env->get_int("HTTP_DEST_ZONE_SHORT_ID", 0); mod_pg_ver = s->info.env->get_int("HTTP_DEST_PG_VER", 0); + rgwx_stat = s->info.args.exists(RGW_SYS_PARAM_PREFIX "stat"); + get_data &= (!rgwx_stat); } /* start gettorrent */ diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 2e09a71410d7d..832bd37a908a4 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -694,19 +694,31 @@ int RGWRESTStreamRWRequest::get_resource(RGWAccessKey& key, map& return 0; } -int RGWRESTStreamRWRequest::complete(string& etag, real_time *mtime, map& attrs) +int RGWRESTStreamRWRequest::complete(string& etag, real_time *mtime, uint64_t *psize, map& attrs) { set_str_from_headers(out_headers, "ETAG", etag); - if (status >= 0 && mtime) { - string mtime_str; - set_str_from_headers(out_headers, "RGWX_MTIME", mtime_str); - if (!mtime_str.empty()) { - int ret = parse_rgwx_mtime(cct, mtime_str, mtime); - if (ret < 0) { - return ret; + if (status >= 0) { + if (mtime) { + string mtime_str; + set_str_from_headers(out_headers, "RGWX_MTIME", mtime_str); + if (!mtime_str.empty()) { + int ret = parse_rgwx_mtime(cct, mtime_str, mtime); + if (ret < 0) { + return ret; + } + } else { + *mtime = real_time(); + } + } + if (psize) { + string size_str; + set_str_from_headers(out_headers, "RGWX_OBJECT_SIZE", size_str); + string err; + *psize = strict_strtoll(size_str.c_str(), 10, &err); + if (!err.empty()) { + ldout(cct, 0) << "ERROR: failed parsing embedded metadata object size (" << size_str << ") to int " << dendl; + return -EIO; } - } else { - *mtime = real_time(); } } diff --git a/src/rgw/rgw_rest_client.h b/src/rgw/rgw_rest_client.h index 6ceeb43f9b646..709e83b8cc912 100644 --- a/src/rgw/rgw_rest_client.h +++ b/src/rgw/rgw_rest_client.h @@ -106,7 +106,7 @@ public: virtual ~RGWRESTStreamRWRequest() {} int get_obj(RGWAccessKey& key, map& extra_headers, rgw_obj& obj); int get_resource(RGWAccessKey& key, map& extra_headers, const string& resource, RGWHTTPManager *mgr = NULL); - int complete(string& etag, real_time *mtime, map& attrs); + int complete(string& etag, real_time *mtime, uint64_t *psize, map& attrs); void set_outbl(bufferlist& _outbl) { outbl.swap(_outbl); diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc index 4ed49d1db34be..68228fbd66cd2 100644 --- a/src/rgw/rgw_rest_conn.cc +++ b/src/rgw/rgw_rest_conn.cc @@ -118,7 +118,7 @@ static void set_header(T val, map& headers, const string& header int RGWRESTConn::get_obj(const rgw_user& uid, req_info *info /* optional */, rgw_obj& obj, const real_time *mod_ptr, const real_time *unmod_ptr, uint32_t mod_zone_id, uint64_t mod_pg_ver, - bool prepend_metadata, bool read_data, + bool prepend_metadata, bool get_op, bool rgwx_stat, RGWGetDataCB *cb, RGWRESTStreamRWRequest **req) { string url; @@ -134,11 +134,14 @@ int RGWRESTConn::get_obj(const rgw_user& uid, req_info *info /* optional */, rgw if (prepend_metadata) { params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "prepend-metadata", self_zone_group)); } + if (rgwx_stat) { + params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "stat", "true")); + } if (!obj.get_instance().empty()) { const string& instance = obj.get_instance(); params.push_back(param_pair_t("versionId", instance)); } - if (read_data) { + if (get_op) { *req = new RGWRESTStreamReadRequest(cct, url, cb, NULL, ¶ms); } else { *req = new RGWRESTStreamHeadRequest(cct, url, cb, NULL, ¶ms); @@ -171,9 +174,10 @@ int RGWRESTConn::get_obj(const rgw_user& uid, req_info *info /* optional */, rgw return (*req)->get_obj(key, extra_headers, obj); } -int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req, string& etag, real_time *mtime, map& attrs) +int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req, string& etag, real_time *mtime, + uint64_t *psize, map& attrs) { - int ret = req->complete(etag, mtime, attrs); + int ret = req->complete(etag, mtime, psize, attrs); delete req; return ret; @@ -215,7 +219,7 @@ int RGWRESTConn::get_resource(const string& resource, string etag; map attrs; - return req.complete(etag, NULL, attrs); + return req.complete(etag, NULL, NULL, attrs); } RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn, @@ -262,7 +266,7 @@ int RGWRESTReadResource::read() string etag; map attrs; - return req.complete(etag, NULL, attrs); + return req.complete(etag, NULL, NULL, attrs); } int RGWRESTReadResource::aio_read() @@ -321,7 +325,7 @@ int RGWRESTPostResource::send(bufferlist& outbl) string etag; map attrs; - return req.complete(etag, NULL, attrs); + return req.complete(etag, NULL, NULL, attrs); } int RGWRESTPostResource::aio_send(bufferlist& outbl) diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index 1ef6385b20b3d..a8d056caa46a9 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -89,9 +89,9 @@ public: int get_obj(const rgw_user& uid, req_info *info /* optional */, rgw_obj& obj, const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr, uint32_t mod_zone_id, uint64_t mod_pg_ver, - bool prepend_metadata, bool read_data, + bool prepend_metadata, bool get_op, bool rgwx_stat, RGWGetDataCB *cb, RGWRESTStreamRWRequest **req); - int complete_request(RGWRESTStreamRWRequest *req, string& etag, ceph::real_time *mtime, map& attrs); + int complete_request(RGWRESTStreamRWRequest *req, string& etag, ceph::real_time *mtime, uint64_t *psize, map& attrs); int get_resource(const string& resource, param_vec_t *extra_params, diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index df16f7db6f870..b2f539d3ce2c2 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -160,6 +160,16 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, if (s->system_request && s->info.args.exists(RGW_SYS_PARAM_PREFIX "prepend-metadata")) { + STREAM_IO(s)->print("Rgwx-Object-Size: %lld\r\n", (long long)total_len); + + if (rgwx_stat) { + /* + * in this case, we're not returning the object's content, only the prepended + * extra metadata + */ + total_len = 0; + } + /* JSON encode object metadata */ JSONFormatter jf; jf.open_object_section("obj_metadata"); -- 2.39.5