From 076a7e7feeb7c79e6c66096369e1f6c95d203d93 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Mon, 21 Dec 2020 20:50:30 +0530 Subject: [PATCH] rgw/CloudTransition: Add documentation Also to avoid object name collisions across various buckets post cloud transition, add bucket name to the object prefix. Signed-off-by: Soumya Koduri --- doc/radosgw/cloud-transition.rst | 323 +++++++++++++++++++++++++++++++ doc/radosgw/index.rst | 1 + src/rgw/rgw_lc.cc | 7 +- src/rgw/rgw_lc_tier.cc | 18 +- 4 files changed, 342 insertions(+), 7 deletions(-) create mode 100644 doc/radosgw/cloud-transition.rst diff --git a/doc/radosgw/cloud-transition.rst b/doc/radosgw/cloud-transition.rst new file mode 100644 index 00000000000..9b756b8ee8c --- /dev/null +++ b/doc/radosgw/cloud-transition.rst @@ -0,0 +1,323 @@ +================ +Cloud Transition +================ + +This feature enables data transition to a remote cloud service as part of `Lifecycle Configuration `__ via `Storage Classes `__. The transition is unidirectional; data cannot be transitioned back from the remote zone. The goal of this feature is to enable data transition to multiple cloud providers. The currently supported cloud providers are those that are compatible with AWS (S3). + +Special storage class of tier type cloud is used to configure the remote cloud object store service to which the data needs to be transitioned to. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool. Any additions or modifications need period commit to get reflected. + +User credentials for the remote cloud object store service need to be configured. Note that source ACLs will not +be preserved. It is possible to map permissions of specific source users to specific destination users. + + +Cloud Storage Class Configuration +--------------------------------- + +:: + + { + "access_key": , + "secret": , + "endpoint": , + "host_style": , + "acls": [ { "type": , + "source_id": , + "dest_id": } ... ], + "target_path": , + "target_storage_class": , + "multipart_sync_threshold": {object_size}, + "multipart_min_part_size": {part_size}, + "retain_object": + } + + +Cloud Transition Specific Configurables: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``access_key`` (string) + +The remote cloud access key that will be used for a specific connection. + +* ``secret`` (string) + +The secret key for the remote cloud service. + +* ``endpoint`` (string) + +URL of remote cloud service endpoint. + +* ``host_style`` (path | virtual) + +Type of host style to be used when accessing remote cloud endpoint (default: ``path``). + +* ``acls`` (array) + +Contains a list of ``acl_mappings``. + +* ``acl_mapping`` (container) + +Each ``acl_mapping`` structure contains ``type``, ``source_id``, and ``dest_id``. These +will define the ACL mutation that will be done on each object. An ACL mutation allows converting source +user id to a destination id. + +* ``type`` (id | email | uri) + +ACL type: ``id`` defines user id, ``email`` defines user by email, and ``uri`` defines user by ``uri`` (group). + +* ``source_id`` (string) + +ID of user in the source zone. + +* ``dest_id`` (string) + +ID of user in the destination. + +* ``target_path`` (string) + +A string that defines how the target path is created. The target path specifies a prefix to which +the source 'bucket-name/object-name' is appended. If not specified the target_path created is "rgwx-${zonegroup}-${storage-class}-cloud-bucket". + +For example: ``target_path = rgwx-archive-${zonegroup}/`` + +* ``target_storage_class`` (string) + +A string that defines the target storage class to which the object transitions to. If not specified, object is transitioned to STANDARD storage class. + +* ``retain_object`` (true | false) + +If true, retains the metadata of the object transitioned to cloud. If false (default), the object is deleted post transition. + + +S3 Specific Configurables: +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Currently cloud transition will only work with backends that are compatible with AWS S3. There are +a few configurables that can be used to tweak its behavior when accessing these cloud services: + +:: + + { + "multipart_sync_threshold": {object_size}, + "multipart_min_part_size": {part_size} + } + + +* ``multipart_sync_threshold`` (integer) + +Objects this size or larger will be transitioned to the cloud using multipart upload. + +* ``multipart_min_part_size`` (integer) + +Minimum parts size to use when transitioning objects using multipart upload. + + +How to Configure +~~~~~~~~~~~~~~~~ + +See `Adding a Storage Class `__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud`` + +Note: Once a storage class is created of ``-tier-type=cloud``, it cannot be later modified to any other storage class type. + +:: + + # radosgw-admin zonegroup placement add --rgw-zonegroup={zone-group-name} \ + --placement-id={placement-id} \ + --storage-class={storage-class-name} \ + --tier-type=cloud + +For example: + +:: + + # radosgw-admin zonegroup placement add --rgw-zonegroup=default \ + --placement-id=default-placement \ + --storage-class=CLOUDTIER --tier-type=cloud + [ + { + "key": "default-placement", + "val": { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "CLOUDTIER", + "STANDARD" + ], + "tier_targets": [ + { + "key": "CLOUDTIER", + "val": { + "storage_class": "CLOUDTIER", + "tier_type": "cloud", + "endpoint": "", + "access_key": "", + "secret": "", + "host_style": "path", + "tier_storage_class": "", + "target_path": "", + "acl_mappings": [], + "multipart_sync_threshold": 33554432, + "multipart_min_part_size": 33554432, + "retain_object": "false" + } + } + ] + } + } + ] + + +The tier configuration can be then done using the following command + +:: + + # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \ + --placement-id={placement-id} \ + --storage-class={storage-class-name} \ + --tier-config={key}={val}[,{key}={val}] + +The ``key`` in the configuration specifies the config variable that needs to be updated, and +the ``val`` specifies its new value. + + +For example: + +:: + + # radosgw-admin zonegroup placement modify --rgw-zonegroup default \ + --placement-id default-placement \ + --storage-class CLOUDTIER \ + --tier-config=endpoint=http://XX.XX.XX.XX:YY,\ + access_key=,secret=, \ + multipart_sync_threshold=44432, \ + multipart_min_part_size=44432, \ + retain_object=true + +Nested values can be accessed using period. For example: + +:: + + # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \ + --placement-id={placement-id} \ + --storage-class={storage-class-name} \ + --tier-config=acls.source_id=${source-id}, \ + acls.dest_id=${dest-id} + + + +Configuration array entries can be accessed by specifying the specific entry to be referenced enclosed +in square brackets, and adding new array entry can be done by using `[]`. +For example, creating a new acl array entry: + +:: + + # radosgw-admin zonegroup placement modify --rgw-zonegroup={zone-group-name} \ + --placement-id={placement-id} \ + --storage-class={storage-class-name} \ + --tier-config=acls[].source_id=${source-id}, \ + acls[${source-id}].dest_id=${dest-id}, \ + acls[${source-id}].type=email + +An entry can be removed by using ``--tier-config-rm={key}``. + +For example, + +:: + + # radosgw-admin zonegroup placement modify --rgw-zonegroup default \ + --placement-id default-placement \ + --storage-class CLOUDTIER \ + --tier-config-rm=acls.source_id=testid + + # radosgw-admin zonegroup placement modify --rgw-zonegroup default \ + --placement-id default-placement \ + --storage-class CLOUDTIER \ + --tier-config-rm=target_path + +The storage class can be removed using the following command + +:: + + # radosgw-admin zonegroup placement rm --rgw-zonegroup={zone-group-name} \ + --placement-id={placement-id} \ + --storage-class={storage-class-name} + +For example, + +:: + + # radosgw-admin zonegroup placement rm --rgw-zonegroup default \ + --placement-id default-placement \ + --storage-class CLOUDTIER + [ + { + "key": "default-placement", + "val": { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "STANDARD" + ] + } + } + ] + +Object modification & Limitations +---------------------------------- + +The cloud storage class once configured can then be used like any other storage class in the bucket lifecyle rules. For example, + +:: + + + CLOUDTIER + .... + .... + + + +Since the transition is unidirectional, while configuring S3 lifecycle rules, the cloud storage class should be specified last among all the storage classes the object transitions to. Subsequent rules (if any) do not apply post transition to the cloud. + +Due to API limitations there is no way to preserve original object modification time and ETag but they get stored as metadata attributes on the destination objects, as shown below: + +:: + + x-amz-meta-rgwx-source: rgw + x-amz-meta-rgwx-source-etag: ed076287532e86365e841e92bfc50d8c + x-amz-meta-rgwx-source-key: lc.txt + x-amz-meta-rgwx-source-mtime: 1608546349.757100363 + x-amz-meta-rgwx-versioned-epoch: 0 + + +By default, post transition, the source object gets deleted. But it is possible to retain its metadata but with updated values (like storage-class and object-size) by setting config option 'retain_object' to true. However GET on those objects shall still fail with 'InvalidObjectState' error. + +For example, +:: + + # s3cmd info s3://bucket/lc.txt + s3://bucket/lc.txt (object): + File size: 12 + Last mod: Mon, 21 Dec 2020 10:25:56 GMT + MIME type: text/plain + Storage: CLOUDTIER + MD5 sum: ed076287532e86365e841e92bfc50d8c + SSE: none + Policy: none + CORS: none + ACL: M. Tester: FULL_CONTROL + x-amz-meta-s3cmd-attrs: atime:1608466266/ctime:1597606156/gid:0/gname:root/md5:ed076287532e86365e841e92bfc50d8c/mode:33188/mtime:1597605793/uid:0/uname:root + + # s3cmd get s3://bucket/lc.txt lc_restore.txt + download: 's3://bucket/lc.txt' -> 'lc_restore.txt' [1 of 1] + ERROR: S3 error: 403 (InvalidObjectState) + + +Since all the objects are transitioned to a single target bucket, bucket versioning is not enabled on the remote endpoint. That means multiple versions of the same object (if any) shall get overwritten based on the order of their transition. + +Future Work +----------- + +* Support s3:RestoreObject operation on cloud transitioned objects. + +* Federation between RGW and Cloud services. + +* Support Object versioning for the transitioned objects diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst index 4b94d340e5c..256172b8de9 100644 --- a/doc/radosgw/index.rst +++ b/doc/radosgw/index.rst @@ -82,4 +82,5 @@ you may write data with one API and retrieve it with the other. S3-select Lua Scripting D3N Data Cache + Cloud Transition diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index d1625486712..7e837b705a1 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1280,7 +1280,7 @@ public: /* XXX: do we need to check for retention/versioning attributes * as done in RGWDeleteObj? */ - ret = oc.store->getRados()->delete_obj(oc.rctx, oc.bucket_info, oc.obj, 0); + ret = oc.store->getRados()->delete_obj(oc.rctx, oc.bucket->get_info(), oc.obj->get_obj(), tier_ctx.bucket_info.versioning_status()); return ret; } @@ -1325,6 +1325,7 @@ public: obj_op.meta.user_data = NULL; obj_op.meta.zones_trace = NULL; obj_op.meta.delete_at = real_time(); + obj_op.meta.olh_epoch = tier_ctx.o.versioned_epoch; RGWObjManifest *pmanifest; @@ -1402,8 +1403,8 @@ public: return ret; } - RGWLCCloudTierCtx tier_ctx(oc.cct, oc.o, oc.store, oc.bucket_info, - oc.obj, oc.rctx, conn, bucket_name, oc.tier.target_storage_class, + RGWLCCloudTierCtx tier_ctx(oc.cct, oc.o, oc.store, oc.bucket->get_info(), + oc.obj->get_obj(), oc.rctx, conn, bucket_name, oc.tier.target_storage_class, &http_manager); tier_ctx.acl_mappings = oc.tier.acl_mappings; tier_ctx.multipart_min_part_size = oc.tier.multipart_min_part_size; diff --git a/src/rgw/rgw_lc_tier.cc b/src/rgw/rgw_lc_tier.cc index 1088c4693de..32a846e8b5f 100644 --- a/src/rgw/rgw_lc_tier.cc +++ b/src/rgw/rgw_lc_tier.cc @@ -523,7 +523,8 @@ class RGWLCStreamObjToCloudPlainCR : public RGWCoroutine { string target_obj_name; target_bucket.name = tier_ctx.target_bucket_name; - target_obj_name = tier_ctx.obj.key.name; // cross check with aws module + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj.key.name; dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); @@ -533,6 +534,8 @@ class RGWLCStreamObjToCloudPlainCR : public RGWCoroutine { reenter(this) { + // tier_ctx.obj.set_atomic(&tier_ctx.rctx); -- might need when updated to zipper SAL + /* Prepare Read from source */ in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); @@ -582,7 +585,8 @@ class RGWLCStreamObjToCloudMultipartPartCR : public RGWCoroutine { off_t end; target_bucket.name = tier_ctx.target_bucket_name; - target_obj_name = tier_ctx.obj.key.name; // cross check with aws module + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj.key.name; dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); @@ -590,6 +594,8 @@ class RGWLCStreamObjToCloudMultipartPartCR : public RGWCoroutine { (rgw::sal::RGWRadosBucket *)(dest_bucket.get()))); reenter(this) { + // tier_ctx.obj.set_atomic(&tier_ctx.rctx); -- might need when updated to zipper SAL + /* Prepare Read from source */ in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); @@ -922,7 +928,10 @@ class RGWLCStreamObjToCloudMultipartCR : public RGWCoroutine { rgw_bucket target_bucket; target_bucket.name = tier_ctx.target_bucket_name; - string target_obj_name = obj.key.name; // cross check with aws module + + string target_obj_name; + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj.key.name; rgw_obj dest_obj(target_bucket, target_obj_name); std::shared_ptr in_crf; rgw_rest_obj rest_obj; @@ -1041,7 +1050,8 @@ int RGWLCCloudCheckCR::operate() { string target_obj_name; target_bucket.name = tier_ctx.target_bucket_name; - target_obj_name = tier_ctx.obj.key.name; // cross check with aws module + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj.key.name; std::shared_ptr dest_bucket; dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); -- 2.39.5