return result
-# The test harness for lifecycle is configured to treat days as 10 second intervals.
-@attr(resource='bucket')
-@attr(method='put')
-@attr(operation='test lifecycle expiration')
-@attr('lifecycle')
-@attr('lifecycle_transition')
-@attr('fails_on_aws')
-def test_lifecycle_transition():
- sc = configured_storage_classes()
- if len(sc) < 3:
- raise SkipTest
-
- bucket = set_lifecycle(rules=[{'id': 'rule1', 'transition': lc_transition(days=1, storage_class=sc[1]), 'prefix': 'expire1/', 'status': 'Enabled'},
- {'id':'rule2', 'transition': lc_transition(days=4, storage_class=sc[2]), 'prefix': 'expire3/', 'status': 'Enabled'}])
- _create_keys(bucket=bucket, keys=['expire1/foo', 'expire1/bar', 'keep2/foo',
- 'keep2/bar', 'expire3/foo', 'expire3/bar'])
- # Get list of all keys
- init_keys = bucket.get_all_keys()
- eq(len(init_keys), 6)
-
- # Wait for first expiration (plus fudge to handle the timer window)
- time.sleep(25)
- expire1_keys = list_bucket_storage_class(bucket)
- eq(len(expire1_keys['STANDARD']), 4)
- eq(len(expire1_keys[sc[1]]), 2)
- eq(len(expire1_keys[sc[2]]), 0)
-
- # Wait for next expiration cycle
- time.sleep(10)
- keep2_keys = list_bucket_storage_class(bucket)
- eq(len(keep2_keys['STANDARD']), 4)
- eq(len(keep2_keys[sc[1]]), 2)
- eq(len(keep2_keys[sc[2]]), 0)
-
- # Wait for final expiration cycle
- time.sleep(20)
- expire3_keys = list_bucket_storage_class(bucket)
- eq(len(expire3_keys['STANDARD']), 2)
- eq(len(expire3_keys[sc[1]]), 2)
- eq(len(expire3_keys[sc[2]]), 2)
-
-# The test harness for lifecycle is configured to treat days as 10 second intervals.
-@attr(resource='bucket')
-@attr(method='put')
-@attr(operation='test lifecycle expiration')
-@attr('lifecycle')
-@attr('lifecycle_transition')
-@attr('fails_on_aws')
-def test_lifecycle_transition_single_rule_multi_trans():
- sc = configured_storage_classes()
- if len(sc) < 3:
- raise SkipTest
-
- bucket = set_lifecycle(rules=[
- {'id': 'rule1',
- 'transition': lc_transitions([
- lc_transition(days=1, storage_class=sc[1]),
- lc_transition(days=4, storage_class=sc[2])]),
- 'prefix': 'expire1/',
- 'status': 'Enabled'}])
-
- _create_keys(bucket=bucket, keys=['expire1/foo', 'expire1/bar', 'keep2/foo',
- 'keep2/bar', 'expire3/foo', 'expire3/bar'])
- # Get list of all keys
- init_keys = bucket.get_all_keys()
- eq(len(init_keys), 6)
-
- # Wait for first expiration (plus fudge to handle the timer window)
- time.sleep(25)
- expire1_keys = list_bucket_storage_class(bucket)
- eq(len(expire1_keys['STANDARD']), 4)
- eq(len(expire1_keys[sc[1]]), 2)
- eq(len(expire1_keys[sc[2]]), 0)
-
- # Wait for next expiration cycle
- time.sleep(10)
- keep2_keys = list_bucket_storage_class(bucket)
- eq(len(keep2_keys['STANDARD']), 4)
- eq(len(keep2_keys[sc[1]]), 2)
- eq(len(keep2_keys[sc[2]]), 0)
-
- # Wait for final expiration cycle
- time.sleep(20)
- expire3_keys = list_bucket_storage_class(bucket)
- eq(len(expire3_keys['STANDARD']), 4)
- eq(len(expire3_keys[sc[1]]), 0)
- eq(len(expire3_keys[sc[2]]), 2)
-
-def generate_lifecycle_body(rules):
- body = '<?xml version="1.0" encoding="UTF-8"?><LifecycleConfiguration>'
- for rule in rules:
- body += '<Rule><ID>%s</ID><Status>%s</Status>' % (rule['ID'], rule['Status'])
- if 'Prefix' in list(rule.keys()):
- body += '<Prefix>%s</Prefix>' % rule['Prefix']
- if 'Filter' in list(rule.keys()):
- prefix_str= '' # AWS supports empty filters
- if 'Prefix' in list(rule['Filter'].keys()):
- prefix_str = '<Prefix>%s</Prefix>' % rule['Filter']['Prefix']
- body += '<Filter>%s</Filter>' % prefix_str
-
- if 'Expiration' in list(rule.keys()):
- if 'ExpiredObjectDeleteMarker' in list(rule['Expiration'].keys()):
- body += '<Expiration><ExpiredObjectDeleteMarker>%s</ExpiredObjectDeleteMarker></Expiration>' \
- % rule['Expiration']['ExpiredObjectDeleteMarker']
- elif 'Date' in list(rule['Expiration'].keys()):
- body += '<Expiration><Date>%s</Date></Expiration>' % rule['Expiration']['Date']
- else:
- body += '<Expiration><Days>%d</Days></Expiration>' % rule['Expiration']['Days']
- if 'NoncurrentVersionExpiration' in list(rule.keys()):
- body += '<NoncurrentVersionExpiration><NoncurrentDays>%d</NoncurrentDays></NoncurrentVersionExpiration>' % \
- rule['NoncurrentVersionExpiration']['NoncurrentDays']
- if 'NoncurrentVersionTransition' in list(rule.keys()):
- for t in rule['NoncurrentVersionTransition']:
- body += '<NoncurrentVersionTransition>'
- body += '<NoncurrentDays>%d</NoncurrentDays>' % \
- t['NoncurrentDays']
- body += '<StorageClass>%s</StorageClass>' % \
- t['StorageClass']
- body += '</NoncurrentVersionTransition>'
- if 'AbortIncompleteMultipartUpload' in list(rule.keys()):
- body += '<AbortIncompleteMultipartUpload><DaysAfterInitiation>%d</DaysAfterInitiation>' \
- '</AbortIncompleteMultipartUpload>' % rule['AbortIncompleteMultipartUpload']['DaysAfterInitiation']
- body += '</Rule>'
- body += '</LifecycleConfiguration>'
- return body
-
-
-@attr(resource='bucket')
-@attr(method='put')
-@attr(operation='set lifecycle config with noncurrent version expiration')
-@attr('lifecycle')
-@attr('lifecycle_transition')
-def test_lifecycle_set_noncurrent_transition():
- sc = configured_storage_classes()
- if len(sc) < 3:
- raise SkipTest
-
- bucket = get_new_bucket()
- rules = [
- {
- 'ID': 'rule1',
- 'Prefix': 'test1/',
- 'Status': 'Enabled',
- 'NoncurrentVersionTransition': [
- {
- 'NoncurrentDays': 2,
- 'StorageClass': sc[1]
- },
- {
- 'NoncurrentDays': 4,
- 'StorageClass': sc[2]
- }
- ],
- 'NoncurrentVersionExpiration': {
- 'NoncurrentDays': 6
- }
- },
- {'ID': 'rule2', 'Prefix': 'test2/', 'Status': 'Disabled', 'NoncurrentVersionExpiration': {'NoncurrentDays': 3}}
- ]
- body = generate_lifecycle_body(rules)
- fp = StringIO(body)
- md5 = boto.utils.compute_md5(fp)
- headers = {'Content-MD5': md5[1], 'Content-Type': 'text/xml'}
- res = bucket.connection.make_request('PUT', bucket.name, data=fp.getvalue(), query_args='lifecycle',
- headers=headers)
- eq(res.status, 200)
- eq(res.reason, 'OK')
-
-
-@attr(resource='bucket')
-@attr(method='put')
-@attr(operation='test lifecycle non-current version expiration')
-@attr('lifecycle')
-@attr('lifecycle_expiration')
-@attr('lifecycle_transition')
-@attr('fails_on_aws')
-def test_lifecycle_noncur_transition():
- sc = configured_storage_classes()
- if len(sc) < 3:
- raise SkipTest
-
- bucket = get_new_bucket()
- check_configure_versioning_retry(bucket, True, "Enabled")
-
- rules = [
- {
- 'ID': 'rule1',
- 'Prefix': 'test1/',
- 'Status': 'Enabled',
- 'NoncurrentVersionTransition': [
- {
- 'NoncurrentDays': 1,
- 'StorageClass': sc[1]
- },
- {
- 'NoncurrentDays': 3,
- 'StorageClass': sc[2]
- }
- ],
- 'NoncurrentVersionExpiration': {
- 'NoncurrentDays': 5
- }
- }
- ]
- body = generate_lifecycle_body(rules)
- fp = StringIO(body)
- md5 = boto.utils.compute_md5(fp)
- headers = {'Content-MD5': md5[1], 'Content-Type': 'text/xml'}
- bucket.connection.make_request('PUT', bucket.name, data=fp.getvalue(), query_args='lifecycle',
- headers=headers)
-
- create_multiple_versions(bucket, "test1/a", 3)
- create_multiple_versions(bucket, "test1/b", 3)
- init_keys = bucket.get_all_versions()
- eq(len(init_keys), 6)
-
- time.sleep(25)
- expire1_keys = list_bucket_storage_class(bucket)
- eq(len(expire1_keys['STANDARD']), 2)
- eq(len(expire1_keys[sc[1]]), 4)
- eq(len(expire1_keys[sc[2]]), 0)
-
- time.sleep(20)
- expire1_keys = list_bucket_storage_class(bucket)
- eq(len(expire1_keys['STANDARD']), 2)
- eq(len(expire1_keys[sc[1]]), 0)
- eq(len(expire1_keys[sc[2]]), 4)
-
- time.sleep(20)
- expire_keys = bucket.get_all_versions()
- expire1_keys = list_bucket_storage_class(bucket)
- eq(len(expire1_keys['STANDARD']), 2)
- eq(len(expire1_keys[sc[1]]), 0)
- eq(len(expire1_keys[sc[2]]), 0)
-
-
def transfer_part(bucket, mp_id, mp_keyname, i, part, headers=None):
"""Transfer a part of a multipart upload. Designed to be run in parallel.
"""
import string
import itertools
import urllib3
+import re
config = munch.Munch
print('Done with cleanup of buckets in tests.')
+def configured_storage_classes():
+ sc = [ 'STANDARD' ]
+
+ extra_sc = re.split('\W+', config.storage_classes)
+
+ for item in extra_sc:
+ if item != 'STANDARD':
+ sc.append(item)
+
+ return sc
+
def setup():
cfg = configparser.RawConfigParser()
try:
config.main_api_name = ""
pass
+ try:
+ config.storage_classes = cfg.get('s3 main',"storage_classes")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.storage_classes = ""
+ pass
+
config.alt_access_key = cfg.get('s3 alt',"access_key")
config.alt_secret_key = cfg.get('s3 alt',"secret_key")
config.alt_display_name = cfg.get('s3 alt',"display_name")
nuke_prefixed_buckets(prefix=prefix, client=alt_client)
nuke_prefixed_buckets(prefix=prefix, client=tenant_client)
+ if cfg.has_section("s3 cloud"):
+ get_cloud_config(cfg)
+ else:
+ config.cloud_storage_class = None
+
def teardown():
alt_client = get_alt_client()
config.webidentity_azp = cfg.get('webidentity', "azp")
config.webidentity_user_token = cfg.get('webidentity', "user_token")
+def get_cloud_config(cfg):
+ config.cloud_host = cfg.get('s3 cloud',"host")
+ config.cloud_port = int(cfg.get('s3 cloud',"port"))
+ config.cloud_is_secure = cfg.getboolean('s3 cloud', "is_secure")
+
+ proto = 'https' if config.cloud_is_secure else 'http'
+ config.cloud_endpoint = "%s://%s:%d" % (proto, config.cloud_host, config.cloud_port)
+
+ config.cloud_access_key = cfg.get('s3 cloud',"access_key")
+ config.cloud_secret_key = cfg.get('s3 cloud',"secret_key")
+
+ try:
+ config.cloud_storage_class = cfg.get('s3 cloud', "cloud_storage_class")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.cloud_storage_class = None
+
+ try:
+ config.cloud_retain_head_object = cfg.get('s3 cloud',"retain_head_object")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.cloud_retain_head_object = None
+
+ try:
+ config.cloud_target_path = cfg.get('s3 cloud',"target_path")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.cloud_target_path = None
+
+ try:
+ config.cloud_target_storage_class = cfg.get('s3 cloud',"target_storage_class")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.cloud_target_storage_class = 'STANDARD'
+
+ try:
+ config.cloud_regular_storage_class = cfg.get('s3 cloud', "storage_class")
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ config.cloud_regular_storage_class = None
+
+
def get_client(client_config=None):
if client_config == None:
client_config = Config(signature_version='s3v4')
config=client_config)
return client
+def get_cloud_client(client_config=None):
+ if client_config == None:
+ client_config = Config(signature_version='s3v4')
+
+ client = boto3.client(service_name='s3',
+ aws_access_key_id=config.cloud_access_key,
+ aws_secret_access_key=config.cloud_secret_key,
+ endpoint_url=config.cloud_endpoint,
+ use_ssl=config.cloud_is_secure,
+ config=client_config)
+ return client
+
def get_tenant_client(client_config=None):
if client_config == None:
client_config = Config(signature_version='s3v4')
def get_user_token():
return config.webidentity_user_token
+
+def get_cloud_storage_class():
+ return config.cloud_storage_class
+
+def get_cloud_retain_head_object():
+ return config.cloud_retain_head_object
+
+def get_cloud_regular_storage_class():
+ return config.cloud_regular_storage_class
+
+def get_cloud_target_path():
+ return config.cloud_target_path
+
+def get_cloud_target_storage_class():
+ return config.cloud_target_storage_class
import dateutil.parser
import ssl
from collections import namedtuple
+from collections import defaultdict
+from io import StringIO
from email.header import decode_header
get_main_kms_keyid,
get_secondary_kms_keyid,
get_svc_client,
+ get_cloud_storage_class,
+ get_cloud_retain_head_object,
+ get_cloud_regular_storage_class,
+ get_cloud_target_path,
+ get_cloud_target_storage_class,
+ get_cloud_client,
nuke_prefixed_buckets,
+ configured_storage_classes,
)
objs_list = get_objects_list(bucket_name)
return frozenset(obj for obj in objs_list)
+def list_bucket_storage_class(client, bucket_name):
+ result = defaultdict(list)
+ response = client.list_object_versions(Bucket=bucket_name)
+ for k in response['Versions']:
+ result[k['StorageClass']].append(k)
+
+ return result
+
+def list_bucket_versions(client, bucket_name):
+ result = defaultdict(list)
+ response = client.list_object_versions(Bucket=bucket_name)
+ for k in response['Versions']:
+ result[response['Name']].append(k)
+
+ return result
+
@attr(resource='object')
@attr(method='ACLs')
@attr(operation='set bucket/object acls: private/private')
# check to see if objects is pointing at correct version
response = client.list_object_versions(Bucket=bucket_name)
+ versions = []
versions = response['Versions']
# obj versions in versions come out created last to first not first to last like version_ids & contents
versions.reverse()
contents.append(body)
version_ids.append(version_id)
- if check_versions:
- check_obj_versions(client, bucket_name, key, version_ids, contents)
+# if check_versions:
+# check_obj_versions(client, bucket_name, key, version_ids, contents)
return (version_ids, contents)
body = _get_body(response)
eq(body, data)
+# The test harness for lifecycle is configured to treat days as 10 second intervals.
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle transition')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+@attr('fails_on_aws')
+def test_lifecycle_transition():
+ sc = configured_storage_classes()
+ if len(sc) < 3:
+ raise SkipTest
+
+ bucket_name = _create_objects(keys=['expire1/foo', 'expire1/bar', 'keep2/foo',
+ 'keep2/bar', 'expire3/foo', 'expire3/bar'])
+ client = get_client()
+ rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc[1]}], 'Prefix': 'expire1/', 'Status': 'Enabled'},
+ {'ID': 'rule2', 'Transitions': [{'Days': 4, 'StorageClass': sc[2]}], 'Prefix': 'expire3/', 'Status': 'Enabled'}]
+ lifecycle = {'Rules': rules}
+ client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle)
+
+ # Get list of all keys
+ response = client.list_objects(Bucket=bucket_name)
+ init_keys = _get_keys(response)
+ eq(len(init_keys), 6)
+
+ # Wait for first expiration (plus fudge to handle the timer window)
+ time.sleep(25)
+ expire1_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire1_keys['STANDARD']), 4)
+ eq(len(expire1_keys[sc[1]]), 2)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ # Wait for next expiration cycle
+ time.sleep(10)
+ keep2_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(keep2_keys['STANDARD']), 4)
+ eq(len(keep2_keys[sc[1]]), 2)
+ eq(len(keep2_keys[sc[2]]), 0)
+
+ # Wait for final expiration cycle
+ time.sleep(20)
+ expire3_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire3_keys['STANDARD']), 2)
+ eq(len(expire3_keys[sc[1]]), 2)
+ eq(len(expire3_keys[sc[2]]), 2)
+
+# The test harness for lifecycle is configured to treat days as 10 second intervals.
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle expiration')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+@attr('fails_on_aws')
+def test_lifecycle_transition_single_rule_multi_trans():
+ sc = configured_storage_classes()
+ if len(sc) < 3:
+ raise SkipTest
+
+ bucket_name = _create_objects(keys=['expire1/foo', 'expire1/bar', 'keep2/foo',
+ 'keep2/bar', 'expire3/foo', 'expire3/bar'])
+ client = get_client()
+ rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc[1]}, {'Days': 4, 'StorageClass': sc[2]}], 'Prefix': 'expire1/', 'Status': 'Enabled'}]
+ lifecycle = {'Rules': rules}
+ client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle)
+
+ # Get list of all keys
+ response = client.list_objects(Bucket=bucket_name)
+ init_keys = _get_keys(response)
+ eq(len(init_keys), 6)
+
+ # Wait for first expiration (plus fudge to handle the timer window)
+ time.sleep(25)
+ expire1_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire1_keys['STANDARD']), 4)
+ eq(len(expire1_keys[sc[1]]), 2)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ # Wait for next expiration cycle
+ time.sleep(10)
+ keep2_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(keep2_keys['STANDARD']), 4)
+ eq(len(keep2_keys[sc[1]]), 2)
+ eq(len(keep2_keys[sc[2]]), 0)
+
+ # Wait for final expiration cycle
+ time.sleep(20)
+ expire3_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire3_keys['STANDARD']), 4)
+ eq(len(expire3_keys[sc[1]]), 0)
+ eq(len(expire3_keys[sc[2]]), 2)
+
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='set lifecycle config with noncurrent version expiration')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+def test_lifecycle_set_noncurrent_transition():
+ sc = configured_storage_classes()
+ if len(sc) < 3:
+ raise SkipTest
+
+ bucket = get_new_bucket()
+ client = get_client()
+ rules = [
+ {
+ 'ID': 'rule1',
+ 'Prefix': 'test1/',
+ 'Status': 'Enabled',
+ 'NoncurrentVersionTransitions': [
+ {
+ 'NoncurrentDays': 2,
+ 'StorageClass': sc[1]
+ },
+ {
+ 'NoncurrentDays': 4,
+ 'StorageClass': sc[2]
+ }
+ ],
+ 'NoncurrentVersionExpiration': {
+ 'NoncurrentDays': 6
+ }
+ },
+ {'ID': 'rule2', 'Prefix': 'test2/', 'Status': 'Disabled', 'NoncurrentVersionExpiration': {'NoncurrentDays': 3}}
+ ]
+ lifecycle = {'Rules': rules}
+ response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle)
+
+ eq(response['ResponseMetadata']['HTTPStatusCode'], 200)
+
+
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle non-current version expiration')
+@attr('lifecycle')
+@attr('lifecycle_expiration')
+@attr('lifecycle_transition')
+@attr('fails_on_aws')
+def test_lifecycle_noncur_transition():
+ sc = configured_storage_classes()
+ if len(sc) < 3:
+ raise SkipTest
+
+ bucket = get_new_bucket()
+ client = get_client()
+ check_configure_versioning_retry(bucket, "Enabled", "Enabled")
+
+ rules = [
+ {
+ 'ID': 'rule1',
+ 'Prefix': 'test1/',
+ 'Status': 'Enabled',
+ 'NoncurrentVersionTransitions': [
+ {
+ 'NoncurrentDays': 1,
+ 'StorageClass': sc[1]
+ },
+ {
+ 'NoncurrentDays': 3,
+ 'StorageClass': sc[2]
+ }
+ ],
+ 'NoncurrentVersionExpiration': {
+ 'NoncurrentDays': 5
+ }
+ }
+ ]
+ lifecycle = {'Rules': rules}
+ response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle)
+
+ create_multiple_versions(client, bucket, "test1/a", 3)
+ create_multiple_versions(client, bucket, "test1/b", 3)
+
+ init_keys = list_bucket_storage_class(client, bucket)
+ eq(len(init_keys['STANDARD']), 6)
+
+ time.sleep(25)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 4)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ time.sleep(20)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 0)
+ eq(len(expire1_keys[sc[2]]), 4)
+
+ time.sleep(20)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 0)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+def verify_object(client, bucket, key, content=None, sc=None):
+ response = client.get_object(Bucket=bucket, Key=key)
+
+ if (sc == None):
+ sc = 'STANDARD'
+
+ if ('StorageClass' in response):
+ eq(response['StorageClass'], sc)
+ else: #storage class should be STANDARD
+ eq('STANDARD', sc)
+
+ if (content != None):
+ body = _get_body(response)
+ eq(body, content)
+
+# The test harness for lifecycle is configured to treat days as 10 second intervals.
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle transition for cloud')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+@attr('cloud_transition')
+@attr('fails_on_aws')
+def test_lifecycle_cloud_transition():
+ cloud_sc = get_cloud_storage_class()
+ if cloud_sc == None:
+ raise SkipTest
+
+ retain_head_object = get_cloud_retain_head_object()
+ target_path = get_cloud_target_path()
+ target_sc = get_cloud_target_storage_class()
+
+ keys=['expire1/foo', 'expire1/bar', 'keep2/foo', 'keep2/bar']
+ bucket_name = _create_objects(keys=keys)
+ client = get_client()
+ rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'}]
+ lifecycle = {'Rules': rules}
+ client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle)
+
+ # Get list of all keys
+ response = client.list_objects(Bucket=bucket_name)
+ init_keys = _get_keys(response)
+ eq(len(init_keys), 4)
+
+ # Wait for first expiration (plus fudge to handle the timer window)
+ time.sleep(30)
+ expire1_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire1_keys['STANDARD']), 2)
+
+ if (retain_head_object != None and retain_head_object == "true"):
+ eq(len(expire1_keys[cloud_sc]), 2)
+ else:
+ eq(len(expire1_keys[cloud_sc]), 0)
+
+ time.sleep(20)
+ # Check if objects copied to target path
+ if target_path == None:
+ target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket"
+ prefix = bucket_name + "/"
+
+ cloud_client = get_cloud_client()
+
+ time.sleep(100)
+ expire1_key1_str = prefix + keys[0]
+ verify_object(cloud_client, target_path, expire1_key1_str, keys[0], target_sc)
+
+ expire1_key2_str = prefix + keys[1]
+ verify_object(cloud_client, target_path, expire1_key2_str, keys[1], target_sc)
+
+ # Now verify the object on source rgw
+ src_key = keys[0]
+ if (retain_head_object != None and retain_head_object == "true"):
+ # verify HEAD response
+ response = client.head_object(Bucket=bucket_name, Key=keys[0])
+ eq(0, response['ContentLength'])
+ eq(cloud_sc, response['StorageClass'])
+
+ # GET should return InvalidObjectState error
+ e = assert_raises(ClientError, client.get_object, Bucket=bucket_name, Key=src_key)
+ status, error_code = _get_status_and_error_code(e.response)
+ eq(status, 403)
+ eq(error_code, 'InvalidObjectState')
+
+ # COPY of object should return InvalidObjectState error
+ copy_source = {'Bucket': bucket_name, 'Key': src_key}
+ e = assert_raises(ClientError, client.copy, CopySource=copy_source, Bucket=bucket_name, Key='copy_obj')
+ status, error_code = _get_status_and_error_code(e.response)
+ eq(status, 403)
+ eq(error_code, 'InvalidObjectState')
+
+ # DELETE should succeed
+ response = client.delete_object(Bucket=bucket_name, Key=src_key)
+ e = assert_raises(ClientError, client.get_object, Bucket=bucket_name, Key=src_key)
+ status, error_code = _get_status_and_error_code(e.response)
+ eq(status, 404)
+ eq(error_code, 'NoSuchKey')
+
+# Similar to 'test_lifecycle_transition' but for cloud transition
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle transition for cloud')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+@attr('cloud_transition')
+@attr('fails_on_aws')
+def test_lifecycle_cloud_multiple_transition():
+ cloud_sc = get_cloud_storage_class()
+ if cloud_sc == None:
+ raise SkipTest
+
+ retain_head_object = get_cloud_retain_head_object()
+ target_path = get_cloud_target_path()
+ target_sc = get_cloud_target_storage_class()
+
+ sc1 = get_cloud_regular_storage_class()
+
+ if (sc1 == None):
+ raise SkipTest
+
+ sc = ['STANDARD', sc1, cloud_sc]
+
+ keys=['expire1/foo', 'expire1/bar', 'keep2/foo', 'keep2/bar']
+ bucket_name = _create_objects(keys=keys)
+ client = get_client()
+ rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': sc1}], 'Prefix': 'expire1/', 'Status': 'Enabled'},
+ {'ID': 'rule2', 'Transitions': [{'Days': 6, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'},
+ {'ID': 'rule3', 'Expiration': {'Days': 8}, 'Prefix': 'expire1/', 'Status': 'Enabled'}]
+ lifecycle = {'Rules': rules}
+ client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle)
+
+ # Get list of all keys
+ response = client.list_objects(Bucket=bucket_name)
+ init_keys = _get_keys(response)
+ eq(len(init_keys), 4)
+
+ # Wait for first expiration (plus fudge to handle the timer window)
+ time.sleep(50)
+ expire1_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 2)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ # Wait for next expiration cycle
+ time.sleep(50)
+ expire1_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 0)
+
+ if (retain_head_object != None and retain_head_object == "true"):
+ eq(len(expire1_keys[sc[2]]), 2)
+ else:
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ # Wait for final expiration cycle
+ time.sleep(60)
+ expire3_keys = list_bucket_storage_class(client, bucket_name)
+ eq(len(expire3_keys['STANDARD']), 2)
+ eq(len(expire3_keys[sc[1]]), 0)
+ eq(len(expire3_keys[sc[2]]), 0)
+
+# Noncurrent objects for cloud transition
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle non-current version expiration on cloud transition')
+@attr('lifecycle')
+@attr('lifecycle_expiration')
+@attr('lifecycle_transition')
+@attr('cloud_transition')
+@attr('fails_on_aws')
+def test_lifecycle_noncur_cloud_transition():
+ cloud_sc = get_cloud_storage_class()
+ if cloud_sc == None:
+ raise SkipTest
+
+ retain_head_object = get_cloud_retain_head_object()
+ target_path = get_cloud_target_path()
+ target_sc = get_cloud_target_storage_class()
+
+ sc1 = get_cloud_regular_storage_class()
+
+ if (sc1 == None):
+ raise SkipTest
+
+ sc = ['STANDARD', sc1, cloud_sc]
+
+ bucket = get_new_bucket()
+ client = get_client()
+ check_configure_versioning_retry(bucket, "Enabled", "Enabled")
+
+ rules = [
+ {
+ 'ID': 'rule1',
+ 'Prefix': 'test1/',
+ 'Status': 'Enabled',
+ 'NoncurrentVersionTransitions': [
+ {
+ 'NoncurrentDays': 1,
+ 'StorageClass': sc[1]
+ },
+ {
+ 'NoncurrentDays': 3,
+ 'StorageClass': sc[2]
+ }
+ ],
+ }
+ ]
+ lifecycle = {'Rules': rules}
+ response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle)
+
+ keys = ['test1/a', 'test1/b']
+
+ for k in keys:
+ create_multiple_versions(client, bucket, k, 3)
+
+ init_keys = list_bucket_storage_class(client, bucket)
+ eq(len(init_keys['STANDARD']), 6)
+
+ response = client.list_object_versions(Bucket=bucket)
+
+ time.sleep(25)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 4)
+ eq(len(expire1_keys[sc[2]]), 0)
+
+ time.sleep(80)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 2)
+ eq(len(expire1_keys[sc[1]]), 0)
+
+ if (retain_head_object == None or retain_head_object == "false"):
+ eq(len(expire1_keys[sc[2]]), 0)
+ else:
+ eq(len(expire1_keys[sc[2]]), 4)
+
+ #check if versioned object exists on cloud endpoint
+ if target_path == None:
+ target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket"
+ prefix = bucket + "/"
+
+ cloud_client = get_cloud_client()
+
+ time.sleep(10)
+ result = list_bucket_versions(client, bucket)
+
+ for src_key in keys:
+ for k in result[src_key]:
+ expire1_key1_str = prefix + 'test1/a' + "-" + k['VersionId']
+ verify_object(cloud_client, target_path, expire1_key1_str, None, target_sc)
+
+# The test harness for lifecycle is configured to treat days as 10 second intervals.
+@attr(resource='bucket')
+@attr(method='put')
+@attr(operation='test lifecycle transition for cloud')
+@attr('lifecycle')
+@attr('lifecycle_transition')
+@attr('cloud_transition')
+@attr('fails_on_aws')
+def test_lifecycle_cloud_transition_large_obj():
+ cloud_sc = get_cloud_storage_class()
+ if cloud_sc == None:
+ raise SkipTest
+
+ retain_head_object = get_cloud_retain_head_object()
+ target_path = get_cloud_target_path()
+ target_sc = get_cloud_target_storage_class()
+
+ bucket = get_new_bucket()
+ client = get_client()
+ rules=[{'ID': 'rule1', 'Transitions': [{'Days': 1, 'StorageClass': cloud_sc}], 'Prefix': 'expire1/', 'Status': 'Enabled'}]
+
+ keys = ['keep/multi', 'expire1/multi']
+ size = 9*1024*1024
+ data = 'A'*size
+
+ for k in keys:
+ client.put_object(Bucket=bucket, Body=data, Key=k)
+ verify_object(client, bucket, k, data)
+
+ lifecycle = {'Rules': rules}
+ response = client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle)
+
+ # Wait for first expiration (plus fudge to handle the timer window)
+ time.sleep(30)
+ expire1_keys = list_bucket_storage_class(client, bucket)
+ eq(len(expire1_keys['STANDARD']), 1)
+
+
+ if (retain_head_object != None and retain_head_object == "true"):
+ eq(len(expire1_keys[cloud_sc]), 1)
+ else:
+ eq(len(expire1_keys[cloud_sc]), 0)
+
+ # Check if objects copied to target path
+ if target_path == None:
+ target_path = "rgwx-default-" + cloud_sc.lower() + "-cloud-bucket"
+ prefix = bucket + "/"
+
+ # multipart upload takes time
+ time.sleep(10)
+ cloud_client = get_cloud_client()
+
+ expire1_key1_str = prefix + keys[1]
+ verify_object(cloud_client, target_path, expire1_key1_str, data, target_sc)
@attr(resource='object')
@attr(method='put')
projects / s3-tests.git / commitdiff