This feature enables data transition to a remote cloud service as part of `Lifecycle Configuration <https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html>`__ via `Storage Classes <https://docs.ceph.com/en/latest/radosgw/placement/#storage-classes>`__. The transition is unidirectional; data cannot be transitioned back from the remote zone. The goal of this feature is to enable data transition to multiple cloud providers. The currently supported cloud providers are those that are compatible with AWS (S3).
-Special storage class of tier type cloud is used to configure the remote cloud object store service to which the data needs to be transitioned to. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool. Any additions or modifications need period commit to get reflected.
+Special storage class of tier type ``cloud-s3`` is used to configure the remote cloud S3 object store service to which the data needs to be transitioned to. These are defined in terms of zonegroup placement targets and unlike regular storage classes, do not need a data pool. Any additions or modifications need period commit to get reflected.
User credentials for the remote cloud object store service need to be configured. Note that source ACLs will not
be preserved. It is possible to map permissions of specific source users to specific destination users.
* ``access_key`` (string)
-The remote cloud access key that will be used for a specific connection.
+The remote cloud S3 access key that will be used for a specific connection.
* ``secret`` (string)
-The secret key for the remote cloud service.
+The secret key for the remote cloud S3 service.
* ``endpoint`` (string)
-URL of remote cloud service endpoint.
+URL of remote cloud S3 service endpoint.
* ``host_style`` (path | virtual)
-Type of host style to be used when accessing remote cloud endpoint (default: ``path``).
+Type of host style to be used when accessing remote cloud S3 endpoint (default: ``path``).
* ``acls`` (array)
How to Configure
~~~~~~~~~~~~~~~~
-See `Adding a Storage Class <https://docs.ceph.com/en/latest/radosgw/placement/#adding-a-storage-class>`__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud``
+See `Adding a Storage Class <https://docs.ceph.com/en/latest/radosgw/placement/#adding-a-storage-class>`__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud-s3``
-Note: Once a storage class is created of ``--tier-type=cloud``, it cannot be later modified to any other storage class type.
+Note: Once a storage class is created of ``--tier-type=cloud-s3``, it cannot be later modified to any other storage class type.
::
# radosgw-admin zonegroup placement add --rgw-zonegroup={zone-group-name} \
--placement-id={placement-id} \
--storage-class={storage-class-name} \
- --tier-type=cloud
+ --tier-type=cloud-s3
For example:
# radosgw-admin zonegroup placement add --rgw-zonegroup=default \
--placement-id=default-placement \
- --storage-class=CLOUDTIER --tier-type=cloud
+ --storage-class=CLOUDTIER --tier-type=cloud-s3
[
{
"key": "default-placement",
{
"key": "CLOUDTIER",
"val": {
+ "tier_type": "cloud-s3",
"storage_class": "CLOUDTIER",
- "tier_type": "cloud",
- "endpoint": "",
- "access_key": "",
- "secret": "",
- "host_style": "path",
- "tier_storage_class": "",
- "target_path": "",
- "acl_mappings": [],
- "multipart_sync_threshold": 33554432,
- "multipart_min_part_size": 33554432,
- "retain_object": "false"
+ "retain_object": "false",
+ "s3": {
+ "endpoint": "",
+ "access_key": "",
+ "secret": "",
+ "host_style": "path",
+ "target_storage_class": "",
+ "target_path": "",
+ "acl_mappings": [],
+ "multipart_sync_threshold": 33554432,
+ "multipart_min_part_size": 33554432
+ }
}
}
]
* Support s3:RestoreObject operation on cloud transitioned objects.
* Federation between RGW and Cloud services.
+
+* Support transition to other cloud provideres (like Azure).
pt = &ptiter->second;
tier_class = true;
} else if (tier_type_specified) {
- if (tier_type == "cloud") {
- /* we support only cloud tier-type for now.
+ if (tier_type == "cloud-s3") {
+ /* we support only cloud-s3 tier-type for now.
* Once set cant be reset. */
tier_class = true;
pt->tier_type = tier_type;
+ pt->storage_class = storage_class;
} else {
cerr << "ERROR: Invalid tier-type specified" << std::endl;
return EINVAL;
::encode_json("tail_placement", tail_placement, f);
::encode_json("tier_type", tier_type, f);
- if (tier_type == "cloud") {
+ if (tier_type == "cloud-s3") {
::encode_json("tier_config", tier_config, f);
}
void RGWZoneGroupPlacementTier::dump(Formatter *f) const
{
- encode_json("storage_class", storage_class, f);
encode_json("tier_type", tier_type, f);
+ encode_json("storage_class", storage_class, f);
+ encode_json("retain_object", retain_object, f);
+
+ if (tier_type == "cloud-s3") {
+ encode_json("s3", t.s3, f);
+ }
+}
+
+void RGWZoneGroupPlacementTierS3::dump(Formatter *f) const
+{
encode_json("endpoint", endpoint, f);
encode_json("access_key", key.id, f);
encode_json("secret", key.key, f);
encode_json("acl_mappings", acl_mappings, f);
encode_json("multipart_sync_threshold", multipart_sync_threshold, f);
encode_json("multipart_min_part_size", multipart_min_part_size, f);
- encode_json("retain_object", retain_object, f);
}
void RGWZoneGroupPlacementTier::decode_json(JSONObj *obj)
{
- JSONDecoder::decode_json("storage_class", storage_class, obj);
JSONDecoder::decode_json("tier_type", tier_type, obj);
+ JSONDecoder::decode_json("storage_class", storage_class, obj);
+ JSONDecoder::decode_json("retain_object", retain_object, obj);
+
+ if (tier_type == "cloud-s3") {
+ JSONDecoder::decode_json("s3", t.s3, obj);
+ }
+}
+
+void RGWZoneGroupPlacementTierS3::decode_json(JSONObj *obj)
+{
JSONDecoder::decode_json("endpoint", endpoint, obj);
JSONDecoder::decode_json("access_key", key.id, obj);
JSONDecoder::decode_json("secret", key.key, obj);
JSONDecoder::decode_json("acl_mappings", acl_mappings, obj);
JSONDecoder::decode_json("multipart_sync_threshold", multipart_sync_threshold, obj);
JSONDecoder::decode_json("multipart_min_part_size", multipart_min_part_size, obj);
- JSONDecoder::decode_json("retain_object", retain_object, obj);
}
void RGWZoneGroupPlacementTarget::dump(Formatter *f) const
/* init */
string id = "cloudid";
- string endpoint=oc.tier.endpoint;
- RGWAccessKey key = oc.tier.key;
- HostStyle host_style = oc.tier.host_style;
- string bucket_name = oc.tier.target_path;
+ string endpoint=oc.tier.t.s3.endpoint;
+ RGWAccessKey key = oc.tier.t.s3.key;
+ HostStyle host_style = oc.tier.t.s3.host_style;
+ string bucket_name = oc.tier.t.s3.target_path;
const RGWZoneGroup& zonegroup = oc.store->svc()->zone->get_zonegroup();
if (bucket_name.empty()) {
RGWLCCloudTierCtx tier_ctx(oc.cct, oc.dpp, oc.o, oc.store, oc.bucket->get_info(),
oc.obj->get_obj(), oc.rctx, conn, bucket_name,
- oc.tier.target_storage_class, &http_manager);
- tier_ctx.acl_mappings = oc.tier.acl_mappings;
- tier_ctx.multipart_min_part_size = oc.tier.multipart_min_part_size;
- tier_ctx.multipart_sync_threshold = oc.tier.multipart_sync_threshold;
+ oc.tier.t.s3.target_storage_class, &http_manager);
+ tier_ctx.acl_mappings = oc.tier.t.s3.acl_mappings;
+ tier_ctx.multipart_min_part_size = oc.tier.t.s3.multipart_min_part_size;
+ tier_ctx.multipart_sync_threshold = oc.tier.t.s3.multipart_sync_threshold;
tier_ctx.storage_class = oc.tier.storage_class;
bool al_tiered = false;
r = get_tier_target(zonegroup, target_placement, target_placement.storage_class, oc.tier);
- if (!r && oc.tier.tier_type == "cloud") {
- ldpp_dout(oc.dpp, 20) << "Found cloud tier: " << target_placement.storage_class << dendl;
+ if (!r && oc.tier.tier_type == "cloud-s3") {
+ ldpp_dout(oc.dpp, 20) << "Found cloud s3 tier: " << target_placement.storage_class << dendl;
r = transition_obj_to_cloud(oc);
if (r < 0) {
ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj to cloud (r=" << r << ")"
{
int r = -1;
+ if (config.exists("retain_object")) {
+ string s = config["retain_object"];
+ if (s == "true") {
+ retain_object = true;
+ } else {
+ retain_object = false;
+ }
+ }
+
+ if (tier_type == "cloud-s3") {
+ r = t.s3.update_params(config);
+ }
+
+ return r;
+}
+
+int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config)
+{
+ int r = -1;
+
if (config.exists("endpoint")) {
endpoint = config["endpoint"];
}
if (config.exists("secret")) {
key.key = config["secret"];
}
- if (config.exists("retain_object")) {
- string s = config["retain_object"];
- if (s == "true") {
- retain_object = true;
- } else {
- retain_object = false;
- }
- }
-
if (config.exists("multipart_sync_threshold")) {
r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold);
if (r < 0) {
return 0;
}
int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config)
+{
+ if (config.exists("retain_object")) {
+ retain_object = false;
+ }
+
+ if (tier_type == "cloud-s3") {
+ t.s3.clear_params(config);
+ }
+
+ return 0;
+}
+
+int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config)
{
if (config.exists("endpoint")) {
endpoint.clear();
if (config.exists("secret")) {
key.key.clear();
}
- if (config.exists("retain_object")) {
- retain_object = false;
- }
if (config.exists("multipart_sync_threshold")) {
multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE;
}
};
WRITE_CLASS_ENCODER(RGWTierACLMapping)
-struct RGWZoneGroupPlacementTier {
+struct RGWZoneGroupPlacementTierS3 {
#define DEFAULT_MULTIPART_SYNC_PART_SIZE (32 * 1024 * 1024)
- std::string storage_class;
- std::string tier_type;
std::string endpoint;
RGWAccessKey key;
HostStyle host_style{PathStyle};
uint64_t multipart_sync_threshold{DEFAULT_MULTIPART_SYNC_PART_SIZE};
uint64_t multipart_min_part_size{DEFAULT_MULTIPART_SYNC_PART_SIZE};
- bool retain_object = false;
-
int update_params(const JSONFormattable& config);
int clear_params(const JSONFormattable& config);
void encode(bufferlist& bl) const {
ENCODE_START(1, 1, bl);
- encode(storage_class, bl);
- encode(tier_type, bl);
encode(endpoint, bl);
encode(key, bl);
string s = (host_style == PathStyle ? "path" : "virtual");
encode(acl_mappings, bl);
encode(multipart_sync_threshold, bl);
encode(multipart_min_part_size, bl);
- encode(retain_object, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::const_iterator& bl) {
DECODE_START(1, bl);
- decode(storage_class, bl);
- decode(tier_type, bl);
decode(endpoint, bl);
decode(key, bl);
string s;
decode(acl_mappings, bl);
decode(multipart_sync_threshold, bl);
decode(multipart_min_part_size, bl);
+ DECODE_FINISH(bl);
+ }
+ void dump(Formatter *f) const;
+ void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWZoneGroupPlacementTierS3)
+
+struct RGWZoneGroupPlacementTier {
+ std::string tier_type;
+ std::string storage_class;
+ bool retain_object = false;
+
+ struct _tier {
+ RGWZoneGroupPlacementTierS3 s3;
+ } t;
+
+ int update_params(const JSONFormattable& config);
+ int clear_params(const JSONFormattable& config);
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(tier_type, bl);
+ encode(storage_class, bl);
+ encode(retain_object, bl);
+ if (tier_type == "cloud-s3") {
+ encode(t.s3, bl);
+ }
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(1, bl);
+ decode(tier_type, bl);
+ decode(storage_class, bl);
decode(retain_object, bl);
+ if (tier_type == "cloud-s3") {
+ decode(t.s3, bl);
+ }
DECODE_FINISH(bl);
}
+
void dump(Formatter *f) const;
void decode_json(JSONObj *obj);
};
projects / ceph.git / commitdiff