From 6753e27daab5135973fa45e5a45a4e8e5fbac101 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Thu, 6 Jun 2024 12:06:49 -0700 Subject: [PATCH] multisite/qa: resolve rebase conflicts Signed-off-by: Shilpa Jagannath --- src/test/rgw/rgw_multi/tests.py | 704 +++++++++++++++----------------- 1 file changed, 334 insertions(+), 370 deletions(-) diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 2b3e207752c07..df9c70a47810a 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -765,38 +765,6 @@ def check_oidc_providers_eq(zone_conn1, zone_conn2): eq(p1, p2) check_oidc_provider_eq(zone_conn1, zone_conn2, p1['Arn']) -def create_zone_bucket(zone): - b_name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, b_name) - bucket = zone.create_bucket(b_name) - return bucket - -def create_object(zone_conn, bucket, objname, content): - k = new_key(zone_conn, bucket.name, objname) - k.set_contents_from_string(content) - -def create_objects(zone_conn, bucket, obj_arr, content): - for objname in obj_arr: - create_object(zone_conn, bucket, objname, content) - -def check_object_exists(bucket, objname, content = None): - k = bucket.get_key(objname) - assert_not_equal(k, None) - if (content != None): - assert_equal(k.get_contents_as_string(encoding='ascii'), content) - -def check_objects_exist(bucket, obj_arr, content = None): - for objname in obj_arr: - check_object_exists(bucket, objname, content) - -def check_object_not_exists(bucket, objname): - k = bucket.get_key(objname) - assert_equal(k, None) - -def check_objects_not_exist(bucket, obj_arr): - for objname in obj_arr: - check_object_not_exists(bucket, objname) - def test_object_sync(): zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) @@ -1568,6 +1536,97 @@ def test_bucket_index_log_trim(): cold_bilog = bilog_list(zone.zone, cold_bucket.name) assert(len(cold_bilog) == 0) +# TODO: disable failing tests temporarily +# until they are fixed + +@attr('fails_with_rgw') +def test_bucket_reshard_index_log_trim(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + zone = zonegroup_conns.rw_zones[0] + + # create a test bucket, upload some objects, and wait for sync + def make_test_bucket(): + name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, name) + bucket = zone.conn.create_bucket(name) + for objname in ('a', 'b', 'c', 'd'): + k = new_key(zone, name, objname) + k.set_contents_from_string('foo') + zonegroup_meta_checkpoint(zonegroup) + zonegroup_bucket_checkpoint(zonegroup_conns, name) + return bucket + + # create a 'test' bucket + test_bucket = make_test_bucket() + + # checking bucket layout before resharding + json_obj_1 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_1['layout']['logs']) == 1) + + first_gen = json_obj_1['layout']['current_index']['gen'] + + before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) + assert(len(before_reshard_bilog) == 4) + + # Resharding the bucket + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '3', + '--yes-i-really-mean-it']) + + # checking bucket layout after 1st resharding + json_obj_2 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_2['layout']['logs']) == 2) + + second_gen = json_obj_2['layout']['current_index']['gen'] + + after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) + assert(len(after_reshard_bilog) == 0) + + # upload more objects + for objname in ('e', 'f', 'g', 'h'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # Resharding the bucket again + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '3', + '--yes-i-really-mean-it']) + + # checking bucket layout after 2nd resharding + json_obj_3 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_3['layout']['logs']) == 3) + + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + bilog_autotrim(zone.zone) + + # checking bucket layout after 1st bilog autotrim + json_obj_4 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_4['layout']['logs']) == 2) + + bilog_autotrim(zone.zone) + + # checking bucket layout after 2nd bilog autotrim + json_obj_5 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_5['layout']['logs']) == 1) + + bilog_autotrim(zone.zone) + + # upload more objects + for objname in ('i', 'j', 'k', 'l'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # verify the bucket has non-empty bilog + test_bilog = bilog_list(zone.zone, test_bucket.name) + assert(len(test_bilog) > 0) + @attr('bucket_reshard') def test_bucket_reshard_incremental(): zonegroup = realm.master_zonegroup() @@ -1664,348 +1723,53 @@ def write_most_shards(zone, bucket_name, num_shards): Write one object to most (but not all) bucket index shards. """ objs = get_bucket_shard_objects(zone.zone, num_shards) - random.shuffle(objs) - del objs[-(len(objs)//10):] - for obj in objs: - k = new_key(zone, bucket_name, obj) - k.set_contents_from_string('foo') - -def reshard_bucket(zone, bucket_name, num_shards): - """ - Reshard a bucket - """ - cmd = ['bucket', 'reshard'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--num-shards', str(num_shards)] - cmd += ['--yes-i-really-mean-it'] - zone.cluster.admin(cmd) - -def get_obj_names(zone, bucket_name, maxobjs): - """ - Get names of objects in a bucket. - """ - cmd = ['bucket', 'list'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--max-entries', str(maxobjs)] - objs_json, _ = zone.cluster.admin(cmd, read_only=True) - objs = json.loads(objs_json) - return [o['name'] for o in objs] - -def bucket_keys_eq(zone1, zone2, bucket_name): - """ - Ensure that two buckets have the same keys, but get the lists through - radosgw-admin rather than S3 so it can be used when radosgw isn't running. - Only works for buckets of 10,000 objects since the tests calling it don't - need more, and the output from bucket list doesn't have an obvious marker - with which to continue. - """ - keys1 = get_obj_names(zone1, bucket_name, 10000) - keys2 = get_obj_names(zone2, bucket_name, 10000) - for key1, key2 in zip_longest(keys1, keys2): - if key1 is None: - log.critical('key=%s is missing from zone=%s', key1.name, - zone1.name) - assert False - if key2 is None: - log.critical('key=%s is missing from zone=%s', key2.name, - zone2.name) - assert False - -def test_role_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - roles, zone_role = create_role_per_zone(zonegroup_conns) - - zonegroup_meta_checkpoint(zonegroup) - - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - -def test_role_delete_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - role_name = gen_role_name() - log.info('create role zone=%s name=%s', zonegroup_conns.master_zone.name, role_name) - policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}" - zonegroup_conns.master_zone.iam_conn.create_role(RoleName=role_name, AssumeRolePolicyDocument=policy_document) - - zonegroup_meta_checkpoint(zonegroup) - - for zone in zonegroup_conns.zones: - log.info(f'checking if zone: {zone.name} has role: {role_name}') - zone.iam_conn.get_role(RoleName=role_name) - log.info(f'success, zone: {zone.name} has role: {role_name}') - - log.info(f"deleting role: {role_name}") - zonegroup_conns.master_zone.iam_conn.delete_role(RoleName=role_name) - zonegroup_meta_checkpoint(zonegroup) - - for zone in zonegroup_conns.zones: - log.info(f'checking if zone: {zone.name} does not have role: {role_name}') - assert_raises(zone.iam_conn.exceptions.NoSuchEntityException, - zone.iam_conn.get_role, RoleName=role_name) - log.info(f'success, zone: {zone.name} does not have role: {role_name}') - -@attr('topic notification') -def test_topic_notification_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_meta_checkpoint(zonegroup) - # let wait for users and other settings to sync across all zones. - time.sleep(config.checkpoint_delay) - # create topics in each zone. - zonegroup_conns = ZonegroupConns(zonegroup) - topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) - log.debug("topic_arns: %s", topic_arns) - - zonegroup_meta_checkpoint(zonegroup) - - # verify topics exists in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - log.debug("topics for zone=%s = %s", conn.name, topic_list) - assert_equal(len(topic_list), len(topic_arns)) - for topic_arn_map in topic_list: - assert_true(topic_arn_map['TopicArn'] in topic_arns) - - # create a bucket - bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - log.debug('created bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # create bucket_notification in each zone. - notification_ids = [] - num = 1 - for zone_conn, topic_arn in zone_topic: - noti_id = "bn" + '-' + run_prefix + '-' + str(num) - notification_ids.append(noti_id) - topic_conf = {'Id': noti_id, - 'TopicArn': topic_arn, - 'Events': ['s3:ObjectCreated:*'] - } - num += 1 - log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) - zone_conn.create_notification(bucket.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - - # verify notifications exists in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - log.debug("notifications for zone=%s = %s", conn.name, notification_list) - assert_equal(len(notification_list), len(topic_arns)) - for notification in notification_list: - assert_true(notification['Id'] in notification_ids) - - # verify bucket_topic mapping - # create a new bucket and subcribe it to first topic. - bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - notif_id = "bn-2" + '-' + run_prefix - topic_conf = {'Id': notif_id, - 'TopicArn': topic_arns[0], - 'Events': ['s3:ObjectCreated:*'] - } - zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - if topic['arn'] == topic_arns[0]: - assert_equal(len(topic['subscribed_buckets']), 2) - assert_true(bucket_2.name in topic['subscribed_buckets']) - else: - assert_equal(len(topic['subscribed_buckets']), 1) - assert_true(bucket.name in topic['subscribed_buckets']) - - # delete the 2nd bucket and verify the mapping is removed. - zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - assert_equal(len(topic['subscribed_buckets']), 1) - '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 - is fixed, as the secondary site bucket instance info is currently not - getting deleted coz of the bug hence the bucket-topic mapping - deletion is not invoked on secondary sites.)''' - break - - # delete notifications - zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) - log.debug('Deleting all notifications for bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # verify notification deleted in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - assert_equal(len(notification_list), 0) - - # delete topics - for zone_conn, topic_arn in zone_topic: - log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) - zone_conn.delete_topic(topic_arn) - zonegroup_meta_checkpoint(zonegroup) - - # verify topics deleted in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - assert_equal(len(topic_list), 0) - -def test_account_metadata_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) - managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - # create user, add access key, user policy, managed policy - iam.create_user(UserName=name) - iam.create_access_key(UserName=name) - iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - # create group, group policy, managed policy, add user to group - iam.create_group(GroupName=name) - iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.add_user_to_group(GroupName=name, UserName=name) - # create role, role policy, managed policy - iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) - iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - # TODO: test oidc provider - #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - - #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') - - iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - iam.delete_role_policy(RoleName=name, PolicyName='Allow') - iam.delete_role(RoleName=name) - - iam.remove_user_from_group(GroupName=name, UserName=name) - iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.delete_group_policy(GroupName=name, PolicyName='Allow') - iam.delete_group(GroupName=name) - - iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - iam.delete_user_policy(UserName=name, PolicyName='Allow') - key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] - iam.delete_access_key(UserName=name, AccessKeyId=key_id) - iam.delete_user(UserName=name) - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) - - -# TODO: disable failing tests temporarily -# until they are fixed - -@attr('fails_with_rgw') -def test_bucket_reshard_index_log_trim(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - zone = zonegroup_conns.rw_zones[0] - - # create a test bucket, upload some objects, and wait for sync - def make_test_bucket(): - name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, name) - bucket = zone.conn.create_bucket(name) - for objname in ('a', 'b', 'c', 'd'): - k = new_key(zone, name, objname) - k.set_contents_from_string('foo') - zonegroup_meta_checkpoint(zonegroup) - zonegroup_bucket_checkpoint(zonegroup_conns, name) - return bucket - - # create a 'test' bucket - test_bucket = make_test_bucket() - - # checking bucket layout before resharding - json_obj_1 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_1['layout']['logs']) == 1) - - first_gen = json_obj_1['layout']['current_index']['gen'] - - before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) - assert(len(before_reshard_bilog) == 4) - - # Resharding the bucket - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '13']) - - # checking bucket layout after 1st resharding - json_obj_2 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_2['layout']['logs']) == 2) - - second_gen = json_obj_2['layout']['current_index']['gen'] - - after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) - assert(len(after_reshard_bilog) == 0) - - # upload more objects - for objname in ('e', 'f', 'g', 'h'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - # Resharding the bucket again - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '15']) - - # checking bucket layout after 2nd resharding - json_obj_3 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_3['layout']['logs']) == 3) - - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - bilog_autotrim(zone.zone) - - # checking bucket layout after 1st bilog autotrim - json_obj_4 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_4['layout']['logs']) == 2) + random.shuffle(objs) + del objs[-(len(objs)//10):] + for obj in objs: + k = new_key(zone, bucket_name, obj) + k.set_contents_from_string('foo') - bilog_autotrim(zone.zone) +def reshard_bucket(zone, bucket_name, num_shards): + """ + Reshard a bucket + """ + cmd = ['bucket', 'reshard'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--num-shards', str(num_shards)] + cmd += ['--yes-i-really-mean-it'] + zone.cluster.admin(cmd) - # checking bucket layout after 2nd bilog autotrim - json_obj_5 = bucket_layout(zone.zone, test_bucket.name) - time.sleep(config.checkpoint_delay) - assert(len(json_obj_5['layout']['logs']) == 1) +def get_obj_names(zone, bucket_name, maxobjs): + """ + Get names of objects in a bucket. + """ + cmd = ['bucket', 'list'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--max-entries', str(maxobjs)] + objs_json, _ = zone.cluster.admin(cmd, read_only=True) + objs = json.loads(objs_json) + return [o['name'] for o in objs] - # upload more objects - for objname in ('i', 'j', 'k', 'l'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) +def bucket_keys_eq(zone1, zone2, bucket_name): + """ + Ensure that two buckets have the same keys, but get the lists through + radosgw-admin rather than S3 so it can be used when radosgw isn't running. + Only works for buckets of 10,000 objects since the tests calling it don't + need more, and the output from bucket list doesn't have an obvious marker + with which to continue. + """ + keys1 = get_obj_names(zone1, bucket_name, 10000) + keys2 = get_obj_names(zone2, bucket_name, 10000) + for key1, key2 in zip_longest(keys1, keys2): + if key1 is None: + log.critical('key=%s is missing from zone=%s', key1.name, + zone1.name) + assert False + if key2 is None: + log.critical('key=%s is missing from zone=%s', key2.name, + zone2.name) + assert False - # verify the bucket has non-empty bilog - test_bilog = bilog_list(zone.zone, test_bucket.name) - assert(len(test_bilog) > 0) @attr('fails_with_rgw') @attr('bucket_reshard') @@ -2517,6 +2281,38 @@ def remove_sync_group_pipe(cluster, group, pipe_id, bucket = None, args = None): assert False, 'failed to remove sync group pipe groupid=%s, pipe_id=%s, src_zones=%s, dest_zones=%s, bucket=%s' % (group, pipe_id, src_zones, dest_zones, bucket) return json.loads(result_json) +def create_zone_bucket(zone): + b_name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, b_name) + bucket = zone.create_bucket(b_name) + return bucket + +def create_object(zone_conn, bucket, objname, content): + k = new_key(zone_conn, bucket.name, objname) + k.set_contents_from_string(content) + +def create_objects(zone_conn, bucket, obj_arr, content): + for objname in obj_arr: + create_object(zone_conn, bucket, objname, content) + +def check_object_exists(bucket, objname, content = None): + k = bucket.get_key(objname) + assert_not_equal(k, None) + if (content != None): + assert_equal(k.get_contents_as_string(encoding='ascii'), content) + +def check_objects_exist(bucket, obj_arr, content = None): + for objname in obj_arr: + check_object_exists(bucket, objname, content) + +def check_object_not_exists(bucket, objname): + k = bucket.get_key(objname) + assert_equal(k, None) + +def check_objects_not_exist(bucket, obj_arr): + for objname in obj_arr: + check_object_not_exists(bucket, objname) + @attr('fails_with_rgw') @attr('sync_policy') def test_sync_policy_config_zonegroup(): @@ -3536,4 +3332,172 @@ def test_sync_flow_symmetrical_zonegroup_all_rgw_down(): finally: start_2nd_rgw(zonegroup) +def test_topic_notification_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_meta_checkpoint(zonegroup) + # let wait for users and other settings to sync across all zones. + time.sleep(config.checkpoint_delay) + # create topics in each zone. + zonegroup_conns = ZonegroupConns(zonegroup) + topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) + log.debug("topic_arns: %s", topic_arns) + + zonegroup_meta_checkpoint(zonegroup) + + # verify topics exists in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + log.debug("topics for zone=%s = %s", conn.name, topic_list) + assert_equal(len(topic_list), len(topic_arns)) + for topic_arn_map in topic_list: + assert_true(topic_arn_map['TopicArn'] in topic_arns) + + # create a bucket + bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + log.debug('created bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # create bucket_notification in each zone. + notification_ids = [] + num = 1 + for zone_conn, topic_arn in zone_topic: + noti_id = "bn" + '-' + run_prefix + '-' + str(num) + notification_ids.append(noti_id) + topic_conf = {'Id': noti_id, + 'TopicArn': topic_arn, + 'Events': ['s3:ObjectCreated:*'] + } + num += 1 + log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) + zone_conn.create_notification(bucket.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + + # verify notifications exists in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + log.debug("notifications for zone=%s = %s", conn.name, notification_list) + assert_equal(len(notification_list), len(topic_arns)) + for notification in notification_list: + assert_true(notification['Id'] in notification_ids) + + # verify bucket_topic mapping + # create a new bucket and subcribe it to first topic. + bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + notif_id = "bn-2" + '-' + run_prefix + topic_conf = {'Id': notif_id, + 'TopicArn': topic_arns[0], + 'Events': ['s3:ObjectCreated:*'] + } + zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + if topic['arn'] == topic_arns[0]: + assert_equal(len(topic['subscribed_buckets']), 2) + assert_true(bucket_2.name in topic['subscribed_buckets']) + else: + assert_equal(len(topic['subscribed_buckets']), 1) + assert_true(bucket.name in topic['subscribed_buckets']) + + # delete the 2nd bucket and verify the mapping is removed. + zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + assert_equal(len(topic['subscribed_buckets']), 1) + '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 + is fixed, as the secondary site bucket instance info is currently not + getting deleted coz of the bug hence the bucket-topic mapping + deletion is not invoked on secondary sites.)''' + break + + # delete notifications + zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) + log.debug('Deleting all notifications for bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # verify notification deleted in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + assert_equal(len(notification_list), 0) + + # delete topics + for zone_conn, topic_arn in zone_topic: + log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) + zone_conn.delete_topic(topic_arn) + zonegroup_meta_checkpoint(zonegroup) + + # verify topics deleted in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + assert_equal(len(topic_list), 0) + +def test_account_metadata_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) + managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + # create user, add access key, user policy, managed policy + iam.create_user(UserName=name) + iam.create_access_key(UserName=name) + iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + # create group, group policy, managed policy, add user to group + iam.create_group(GroupName=name) + iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.add_user_to_group(GroupName=name, UserName=name) + # create role, role policy, managed policy + iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) + iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + # TODO: test oidc provider + #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + + #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') + + iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + iam.delete_role_policy(RoleName=name, PolicyName='Allow') + iam.delete_role(RoleName=name) + + iam.remove_user_from_group(GroupName=name, UserName=name) + iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.delete_group_policy(GroupName=name, PolicyName='Allow') + iam.delete_group(GroupName=name) + + iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + iam.delete_user_policy(UserName=name, PolicyName='Allow') + key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] + iam.delete_access_key(UserName=name, AccessKeyId=key_id) + iam.delete_user(UserName=name) + + realm_meta_checkpoint(realm) + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) -- 2.39.5