From: lvshanchun Date: Wed, 29 Nov 2017 08:28:05 +0000 (+0800) Subject: rgw: virtual hosted-style support X-Git-Tag: v13.1.0~270^2~41 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2d8f21c7e9bca99b9a00b93a11db70c4cb7670e4;p=ceph.git rgw: virtual hosted-style support add host-style field in tier-config to specify the related zone's hosted-style used in request from RGW, if this config is not specified, path hosted-style will be used as default. Signed-off-by: lvshanchun --- diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index e6f59a86f437..5b19d16130ee 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -283,6 +283,11 @@ enum RGWObjCategory { RGW_OBJ_CATEGORY_MULTIMETA = 3, }; +enum HostStyle { + PathStyle = 0, + VirtualStyle = 1, +}; + /** Store error returns for output at a different point in the program */ struct rgw_err { rgw_err(); diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 995d211fb219..01d6674e1ca1 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -438,9 +438,24 @@ static void add_grants_headers(map& grants, RGWEnv& env, map& remote_endpoints) + const list& remote_endpoints, + HostStyle _host_style) : cct(_cct), endpoints(remote_endpoints.begin(), remote_endpoints.end()), - remote_id(_remote_id) + remote_id(_remote_id), host_style(_host_style) { if (store) { key = store->get_zone_params().system_key; @@ -22,11 +23,12 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWRados *store, RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& remote_endpoints, - RGWAccessKey _cred) + RGWAccessKey _cred, + HostStyle _host_style) : cct(_cct), endpoints(remote_endpoints.begin(), remote_endpoints.end()), key(std::move(_cred)), - remote_id(_remote_id) + remote_id(_remote_id), host_style(_host_style) { if (store) { self_zone_group = store->get_zonegroup().get_id(); @@ -126,7 +128,7 @@ int RGWRESTConn::put_obj_send_init(rgw_obj& obj, const rgw_http_param_pair *extr append_param_list(params, extra_params); } - RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, ¶ms); + RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, ¶ms, host_style); wr->send_init(obj); *req = wr; return 0; @@ -143,7 +145,7 @@ int RGWRESTConn::put_obj_async(const rgw_user& uid, rgw_obj& obj, uint64_t obj_s param_vec_t params; populate_params(params, &uid, self_zone_group); - RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, ¶ms); + RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, ¶ms, host_style); ret = wr->put_obj_init(key, obj, obj_size, attrs, send); if (ret < 0) { delete wr; @@ -233,7 +235,7 @@ int RGWRESTConn::get_obj(const rgw_obj& obj, const get_obj_params& in_params, bo params.push_back(param_pair_t("versionId", instance)); } if (in_params.get_op) { - *req = new RGWRESTStreamReadRequest(cct, url, in_params.cb, NULL, ¶ms); + *req = new RGWRESTStreamReadRequest(cct, url, in_params.cb, NULL, ¶ms, host_style); } else { *req = new RGWRESTStreamHeadRequest(cct, url, in_params.cb, NULL, ¶ms); } @@ -321,7 +323,7 @@ int RGWRESTConn::get_resource(const string& resource, RGWStreamIntoBufferlist cb(bl); - RGWRESTStreamReadRequest req(cct, url, &cb, NULL, ¶ms); + RGWRESTStreamReadRequest req(cct, url, &cb, NULL, ¶ms, host_style); map headers; if (extra_headers) { @@ -405,7 +407,7 @@ RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn, RGWHTTPManager *_mgr) : cct(_conn->get_ctx()), conn(_conn), method(_method), resource(_resource), params(make_param_list(pp)), cb(bl), mgr(_mgr), - req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL) + req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL, _conn->get_host_style()) { init_common(extra_headers); } @@ -417,7 +419,7 @@ RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn, param_vec_t *extra_headers, RGWHTTPManager *_mgr) : cct(_conn->get_ctx()), conn(_conn), method(_method), resource(_resource), params(params), - cb(bl), mgr(_mgr), req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL) + cb(bl), mgr(_mgr), req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL, _conn->get_host_style()) { init_common(extra_headers); } diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index 1385e870633a..76a462fa09e1 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -62,12 +62,14 @@ class RGWRESTConn RGWAccessKey key; string self_zone_group; string remote_id; + HostStyle host_style; std::atomic counter = { 0 }; public: - RGWRESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints); - RGWRESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, RGWAccessKey _cred); + RGWRESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, HostStyle _host_style = PathStyle); + RGWRESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle); + // custom move needed for atomic RGWRESTConn(RGWRESTConn&& other); RGWRESTConn& operator=(RGWRESTConn&& other); @@ -85,6 +87,10 @@ public: return key; } + HostStyle get_host_style() { + return host_style; + } + CephContext *get_ctx() { return cct; } @@ -170,11 +176,11 @@ class S3RESTConn : public RGWRESTConn { public: - S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints) : - RGWRESTConn(_cct, store, _remote_id, endpoints) {} + S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, HostStyle _host_style = PathStyle) : + RGWRESTConn(_cct, store, _remote_id, endpoints, _host_style) {} - S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, RGWAccessKey _cred): - RGWRESTConn(_cct, store, _remote_id, endpoints, _cred) {} + S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle): + RGWRESTConn(_cct, store, _remote_id, endpoints, _cred, _host_style) {} ~S3RESTConn() override = default; void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup) override { diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/rgw_sync_module_aws.cc index 30d3aa74d952..a526374582c7 100644 --- a/src/rgw/rgw_sync_module_aws.cc +++ b/src/rgw/rgw_sync_module_aws.cc @@ -29,7 +29,6 @@ static string aws_bucket_name(const RGWBucketInfo& bucket_info, bool user_bucket } static string aws_object_name(const RGWBucketInfo& bucket_info, const rgw_obj_key&key, bool user_buckets=false){ - string bucket_name = aws_bucket_name(bucket_info, user_buckets); string object_name; object_name += bucket_info.owner.to_str() + "/"; object_name += bucket_info.bucket.name + "/" + key.name; @@ -44,6 +43,8 @@ static string obj_to_aws_path(const rgw_obj& obj) struct AWSSyncConfig { string s3_endpoint; RGWAccessKey key; + HostStyle host_style; + uint64_t multipart_sync_threshold{DEFAULT_MULTIPART_SYNC_PART_SIZE}; uint64_t multipart_min_part_size{DEFAULT_MULTIPART_SYNC_PART_SIZE}; @@ -757,7 +758,6 @@ class RGWAWSHandleRemoteObjCBCR: public RGWStatRemoteObjCBCR { unordered_map bucket_created; string target_bucket_name; rgw_rest_obj rest_obj; - string obj_path; int ret{0}; uint32_t src_zone_short_id{0}; @@ -799,8 +799,6 @@ public: return set_cr_error(-EINVAL); } - obj_path = bucket_info.bucket.name + "/" + key.name; - target_bucket_name = aws_bucket_name(bucket_info); if (bucket_created.find(target_bucket_name) == bucket_created.end()){ yield { @@ -918,7 +916,8 @@ public: sync_env->store, instance.id, { instance.conf.s3_endpoint }, - instance.conf.key)); + instance.conf.key, + instance.conf.host_style)); } ~RGWAWSDataSyncModule() {} @@ -971,6 +970,17 @@ int RGWAWSSyncModule::create_instance(CephContext *cct, const JSONFormattable& c conf.s3_endpoint = config["s3_endpoint"]; + i = config.find("host_style"); + string host_style_str = config["host_style"]; + + if (host_style_str != "virtual") { + conf.host_style = PathStyle; + } else { + conf.host_style = VirtualStyle; + } + + conf.bucket_suffix = config["bucket_suffix"]; + string access_key = config["access_key"]; string secret = config["secret"];