From: Soumya Koduri Date: Fri, 26 Feb 2021 16:48:52 +0000 (+0530) Subject: rgw/CloudTransition: Change tier-type to cloud-s3 X-Git-Tag: v17.1.0~411^2~5 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9a2c48a5202946da766c3e12979c425176c14506;p=ceph-ci.git rgw/CloudTransition: Change tier-type to cloud-s3 Currently the transition is supported to cloud providers that are compatible with AWS/S3. Hence change the tier-type to cloud-s3 to configure the S3 style endpoint details. Signed-off-by: Soumya Koduri --- diff --git a/doc/radosgw/cloud-transition.rst b/doc/radosgw/cloud-transition.rst index 64453aac36e..663ceba7aa8 100644 --- a/doc/radosgw/cloud-transition.rst +++ b/doc/radosgw/cloud-transition.rst @@ -4,7 +4,7 @@ Cloud Transition This feature enables data transition to a remote cloud service as part of `Lifecycle Configuration `__ via `Storage Classes `__. The transition is unidirectional; data cannot be transitioned back from the remote zone. The goal of this feature is to enable data transition to multiple cloud providers. The currently supported cloud providers are those that are compatible with AWS (S3). -Special storage class of tier type cloud is used to configure the remote cloud object store service to which the data needs to be transitioned to. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool. Any additions or modifications need period commit to get reflected. +Special storage class of tier type ``cloud-s3`` is used to configure the remote cloud S3 object store service to which the data needs to be transitioned to. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool. Any additions or modifications need period commit to get reflected. User credentials for the remote cloud object store service need to be configured. Note that source ACLs will not be preserved. It is possible to map permissions of specific source users to specific destination users. @@ -36,19 +36,19 @@ Cloud Transition Specific Configurables: * ``access_key`` (string) -The remote cloud access key that will be used for a specific connection. +The remote cloud S3 access key that will be used for a specific connection. * ``secret`` (string) -The secret key for the remote cloud service. +The secret key for the remote cloud S3 service. * ``endpoint`` (string) -URL of remote cloud service endpoint. +URL of remote cloud S3 service endpoint. * ``host_style`` (path | virtual) -Type of host style to be used when accessing remote cloud endpoint (default: ``path``). +Type of host style to be used when accessing remote cloud S3 endpoint (default: ``path``). * ``acls`` (array) @@ -114,16 +114,16 @@ Minimum parts size to use when transitioning objects using multipart upload. How to Configure ~~~~~~~~~~~~~~~~ -See `Adding a Storage Class `__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud`` +See `Adding a Storage Class `__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud-s3`` -Note: Once a storage class is created of ``--tier-type=cloud``, it cannot be later modified to any other storage class type. +Note: Once a storage class is created of ``--tier-type=cloud-s3``, it cannot be later modified to any other storage class type. :: # radosgw-admin zonegroup placement add --rgw-zonegroup={zone-group-name} \ --placement-id={placement-id} \ --storage-class={storage-class-name} \ - --tier-type=cloud + --tier-type=cloud-s3 For example: @@ -131,7 +131,7 @@ For example: # radosgw-admin zonegroup placement add --rgw-zonegroup=default \ --placement-id=default-placement \ - --storage-class=CLOUDTIER --tier-type=cloud + --storage-class=CLOUDTIER --tier-type=cloud-s3 [ { "key": "default-placement", @@ -146,18 +146,20 @@ For example: { "key": "CLOUDTIER", "val": { + "tier_type": "cloud-s3", "storage_class": "CLOUDTIER", - "tier_type": "cloud", - "endpoint": "", - "access_key": "", - "secret": "", - "host_style": "path", - "tier_storage_class": "", - "target_path": "", - "acl_mappings": [], - "multipart_sync_threshold": 33554432, - "multipart_min_part_size": 33554432, - "retain_object": "false" + "retain_object": "false", + "s3": { + "endpoint": "", + "access_key": "", + "secret": "", + "host_style": "path", + "target_storage_class": "", + "target_path": "", + "acl_mappings": [], + "multipart_sync_threshold": 33554432, + "multipart_min_part_size": 33554432 + } } } ] @@ -325,3 +327,5 @@ Future Work * Support s3:RestoreObject operation on cloud transitioned objects. * Federation between RGW and Cloud services. + +* Support transition to other cloud provideres (like Azure). diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index ddf9b10d7e6..d734e4960a8 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -5062,11 +5062,12 @@ int main(int argc, const char **argv) pt = &ptiter->second; tier_class = true; } else if (tier_type_specified) { - if (tier_type == "cloud") { - /* we support only cloud tier-type for now. + if (tier_type == "cloud-s3") { + /* we support only cloud-s3 tier-type for now. * Once set cant be reset. */ tier_class = true; pt->tier_type = tier_type; + pt->storage_class = storage_class; } else { cerr << "ERROR: Invalid tier-type specified" << std::endl; return EINVAL; diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 913ad956d07..b234e17165d 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -153,7 +153,7 @@ void RGWObjManifest::dump(Formatter *f) const ::encode_json("tail_placement", tail_placement, f); ::encode_json("tier_type", tier_type, f); - if (tier_type == "cloud") { + if (tier_type == "cloud-s3") { ::encode_json("tier_config", tier_config, f); } @@ -1451,8 +1451,17 @@ void RGWTierACLMapping::decode_json(JSONObj *obj) void RGWZoneGroupPlacementTier::dump(Formatter *f) const { - encode_json("storage_class", storage_class, f); encode_json("tier_type", tier_type, f); + encode_json("storage_class", storage_class, f); + encode_json("retain_object", retain_object, f); + + if (tier_type == "cloud-s3") { + encode_json("s3", t.s3, f); + } +} + +void RGWZoneGroupPlacementTierS3::dump(Formatter *f) const +{ encode_json("endpoint", endpoint, f); encode_json("access_key", key.id, f); encode_json("secret", key.key, f); @@ -1463,13 +1472,21 @@ void RGWZoneGroupPlacementTier::dump(Formatter *f) const encode_json("acl_mappings", acl_mappings, f); encode_json("multipart_sync_threshold", multipart_sync_threshold, f); encode_json("multipart_min_part_size", multipart_min_part_size, f); - encode_json("retain_object", retain_object, f); } void RGWZoneGroupPlacementTier::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("storage_class", storage_class, obj); JSONDecoder::decode_json("tier_type", tier_type, obj); + JSONDecoder::decode_json("storage_class", storage_class, obj); + JSONDecoder::decode_json("retain_object", retain_object, obj); + + if (tier_type == "cloud-s3") { + JSONDecoder::decode_json("s3", t.s3, obj); + } +} + +void RGWZoneGroupPlacementTierS3::decode_json(JSONObj *obj) +{ JSONDecoder::decode_json("endpoint", endpoint, obj); JSONDecoder::decode_json("access_key", key.id, obj); JSONDecoder::decode_json("secret", key.key, obj); @@ -1485,7 +1502,6 @@ void RGWZoneGroupPlacementTier::decode_json(JSONObj *obj) JSONDecoder::decode_json("acl_mappings", acl_mappings, obj); JSONDecoder::decode_json("multipart_sync_threshold", multipart_sync_threshold, obj); JSONDecoder::decode_json("multipart_min_part_size", multipart_min_part_size, obj); - JSONDecoder::decode_json("retain_object", retain_object, obj); } void RGWZoneGroupPlacementTarget::dump(Formatter *f) const diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index e3930c0361a..34111476485 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1376,10 +1376,10 @@ public: /* init */ string id = "cloudid"; - string endpoint=oc.tier.endpoint; - RGWAccessKey key = oc.tier.key; - HostStyle host_style = oc.tier.host_style; - string bucket_name = oc.tier.target_path; + string endpoint=oc.tier.t.s3.endpoint; + RGWAccessKey key = oc.tier.t.s3.key; + HostStyle host_style = oc.tier.t.s3.host_style; + string bucket_name = oc.tier.t.s3.target_path; const RGWZoneGroup& zonegroup = oc.store->svc()->zone->get_zonegroup(); if (bucket_name.empty()) { @@ -1403,10 +1403,10 @@ public: RGWLCCloudTierCtx tier_ctx(oc.cct, oc.dpp, oc.o, oc.store, oc.bucket->get_info(), oc.obj->get_obj(), oc.rctx, conn, bucket_name, - oc.tier.target_storage_class, &http_manager); - tier_ctx.acl_mappings = oc.tier.acl_mappings; - tier_ctx.multipart_min_part_size = oc.tier.multipart_min_part_size; - tier_ctx.multipart_sync_threshold = oc.tier.multipart_sync_threshold; + oc.tier.t.s3.target_storage_class, &http_manager); + tier_ctx.acl_mappings = oc.tier.t.s3.acl_mappings; + tier_ctx.multipart_min_part_size = oc.tier.t.s3.multipart_min_part_size; + tier_ctx.multipart_sync_threshold = oc.tier.t.s3.multipart_sync_threshold; tier_ctx.storage_class = oc.tier.storage_class; bool al_tiered = false; @@ -1463,8 +1463,8 @@ public: r = get_tier_target(zonegroup, target_placement, target_placement.storage_class, oc.tier); - if (!r && oc.tier.tier_type == "cloud") { - ldpp_dout(oc.dpp, 20) << "Found cloud tier: " << target_placement.storage_class << dendl; + if (!r && oc.tier.tier_type == "cloud-s3") { + ldpp_dout(oc.dpp, 20) << "Found cloud s3 tier: " << target_placement.storage_class << dendl; r = transition_obj_to_cloud(oc); if (r < 0) { ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj to cloud (r=" << r << ")" diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index f842cfe929a..5e28704e91e 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -2099,6 +2099,26 @@ int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) { int r = -1; + if (config.exists("retain_object")) { + string s = config["retain_object"]; + if (s == "true") { + retain_object = true; + } else { + retain_object = false; + } + } + + if (tier_type == "cloud-s3") { + r = t.s3.update_params(config); + } + + return r; +} + +int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config) +{ + int r = -1; + if (config.exists("endpoint")) { endpoint = config["endpoint"]; } @@ -2123,15 +2143,6 @@ int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) if (config.exists("secret")) { key.key = config["secret"]; } - if (config.exists("retain_object")) { - string s = config["retain_object"]; - if (s == "true") { - retain_object = true; - } else { - retain_object = false; - } - } - if (config.exists("multipart_sync_threshold")) { r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold); if (r < 0) { @@ -2167,6 +2178,19 @@ int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) return 0; } int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) +{ + if (config.exists("retain_object")) { + retain_object = false; + } + + if (tier_type == "cloud-s3") { + t.s3.clear_params(config); + } + + return 0; +} + +int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config) { if (config.exists("endpoint")) { endpoint.clear(); @@ -2187,9 +2211,6 @@ int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) if (config.exists("secret")) { key.key.clear(); } - if (config.exists("retain_object")) { - retain_object = false; - } if (config.exists("multipart_sync_threshold")) { multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; } diff --git a/src/rgw/rgw_zone.h b/src/rgw/rgw_zone.h index 78ff3bb663b..ae527403538 100644 --- a/src/rgw/rgw_zone.h +++ b/src/rgw/rgw_zone.h @@ -725,10 +725,8 @@ struct RGWTierACLMapping { }; WRITE_CLASS_ENCODER(RGWTierACLMapping) -struct RGWZoneGroupPlacementTier { +struct RGWZoneGroupPlacementTierS3 { #define DEFAULT_MULTIPART_SYNC_PART_SIZE (32 * 1024 * 1024) - std::string storage_class; - std::string tier_type; std::string endpoint; RGWAccessKey key; HostStyle host_style{PathStyle}; @@ -741,15 +739,11 @@ struct RGWZoneGroupPlacementTier { uint64_t multipart_sync_threshold{DEFAULT_MULTIPART_SYNC_PART_SIZE}; uint64_t multipart_min_part_size{DEFAULT_MULTIPART_SYNC_PART_SIZE}; - bool retain_object = false; - int update_params(const JSONFormattable& config); int clear_params(const JSONFormattable& config); void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); - encode(storage_class, bl); - encode(tier_type, bl); encode(endpoint, bl); encode(key, bl); string s = (host_style == PathStyle ? "path" : "virtual"); @@ -759,14 +753,11 @@ struct RGWZoneGroupPlacementTier { encode(acl_mappings, bl); encode(multipart_sync_threshold, bl); encode(multipart_min_part_size, bl); - encode(retain_object, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { DECODE_START(1, bl); - decode(storage_class, bl); - decode(tier_type, bl); decode(endpoint, bl); decode(key, bl); string s; @@ -781,9 +772,47 @@ struct RGWZoneGroupPlacementTier { decode(acl_mappings, bl); decode(multipart_sync_threshold, bl); decode(multipart_min_part_size, bl); + DECODE_FINISH(bl); + } + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; +WRITE_CLASS_ENCODER(RGWZoneGroupPlacementTierS3) + +struct RGWZoneGroupPlacementTier { + std::string tier_type; + std::string storage_class; + bool retain_object = false; + + struct _tier { + RGWZoneGroupPlacementTierS3 s3; + } t; + + int update_params(const JSONFormattable& config); + int clear_params(const JSONFormattable& config); + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(tier_type, bl); + encode(storage_class, bl); + encode(retain_object, bl); + if (tier_type == "cloud-s3") { + encode(t.s3, bl); + } + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(tier_type, bl); + decode(storage_class, bl); decode(retain_object, bl); + if (tier_type == "cloud-s3") { + decode(t.s3, bl); + } DECODE_FINISH(bl); } + void dump(Formatter *f) const; void decode_json(JSONObj *obj); };