From 3e99f4b4520a9e1c5452d3b95ec85a84fc0d940a Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 16 Feb 2024 10:36:21 -0500 Subject: [PATCH] test/rgw/multisite: test sync of iam-related metadata use boto3 for iam connection Signed-off-by: Casey Bodley --- qa/tasks/rgw_multisite_tests.py | 9 +- src/test/rgw/rgw_multi/conn.py | 13 +- src/test/rgw/rgw_multi/multisite.py | 12 +- src/test/rgw/rgw_multi/tests.py | 221 ++++++++++++++++++++++++++-- src/test/rgw/test_multi.py | 7 +- 5 files changed, 229 insertions(+), 33 deletions(-) diff --git a/qa/tasks/rgw_multisite_tests.py b/qa/tasks/rgw_multisite_tests.py index 888a37181690f..822cbcf7910c9 100644 --- a/qa/tasks/rgw_multisite_tests.py +++ b/qa/tasks/rgw_multisite_tests.py @@ -69,11 +69,12 @@ class RGWMultisiteTests(Task): from rgw_multi import multisite, tests - # create the test user + # create test account/user log.info('creating test user..') - user = multisite.User('rgw-multisite-test-user') - user.create(master_zone, ['--display-name', 'Multisite Test User', - '--gen-access-key', '--gen-secret', '--caps', 'roles=*']) + user = multisite.User('rgw-multisite-test-user', account='RGW11111111111111111') + master_zone.cluster.admin(['account', 'create', '--account-id', user.account]) + user.create(master_zone, ['--display-name', 'TestUser', + '--gen-access-key', '--gen-secret']) config = self.config.get('config', {}) tests.init_multi(realm, user, tests.Config(**config)) diff --git a/src/test/rgw/rgw_multi/conn.py b/src/test/rgw/rgw_multi/conn.py index 0ef66b0c4ca94..3edb75111258f 100644 --- a/src/test/rgw/rgw_multi/conn.py +++ b/src/test/rgw/rgw_multi/conn.py @@ -30,15 +30,18 @@ def get_gateway_secure_connection(gateway, credentials): calling_format = boto.s3.connection.OrdinaryCallingFormat()) return gateway.secure_connection -def get_gateway_iam_connection(gateway, credentials): +def get_gateway_iam_connection(gateway, credentials, region): """ connect to iam api of the given gateway """ if gateway.iam_connection is None: - gateway.iam_connection = boto.connect_iam( + endpoint = f'http://{gateway.host}:{gateway.port}' + print(endpoint) + gateway.iam_connection = boto3.client( + service_name = 'iam', aws_access_key_id = credentials.access_key, aws_secret_access_key = credentials.secret, - host = gateway.host, - port = gateway.port, - is_secure = False) + endpoint_url = endpoint, + region_name=region, + use_ssl = False) return gateway.iam_connection diff --git a/src/test/rgw/rgw_multi/multisite.py b/src/test/rgw/rgw_multi/multisite.py index 8642ea3a57cee..4f61425684f0f 100644 --- a/src/test/rgw/rgw_multi/multisite.py +++ b/src/test/rgw/rgw_multi/multisite.py @@ -191,17 +191,16 @@ class ZoneConn(object): self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials) self.secure_conn = get_gateway_secure_connection(self.zone.gateways[0], self.credentials) - self.iam_conn = get_gateway_iam_connection(self.zone.gateways[0], self.credentials) region = "" if self.zone.zonegroup is None else self.zone.zonegroup.name + self.iam_conn = get_gateway_iam_connection(self.zone.gateways[0], self.credentials, region) self.s3_client = get_gateway_s3_client(self.zone.gateways[0], self.credentials, region) - self.sns_client = get_gateway_sns_client(self.zone.gateways[0], self.credentials,region) + self.sns_client = get_gateway_sns_client(self.zone.gateways[0], self.credentials, region) # create connections for the rest of the gateways (if exist) for gw in list(self.zone.gateways): get_gateway_connection(gw, self.credentials) get_gateway_secure_connection(gw, self.credentials) - - get_gateway_iam_connection(gw, self.credentials) + get_gateway_iam_connection(gw, self.credentials, region) def get_connection(self): @@ -369,10 +368,11 @@ class Credentials: return ['--access-key', self.access_key, '--secret', self.secret] class User(SystemObject): - def __init__(self, uid, data = None, name = None, credentials = None, tenant = None): + def __init__(self, uid, data = None, name = None, credentials = None, tenant = None, account = None): self.name = name self.credentials = credentials or [] self.tenant = tenant + self.account = account super(User, self).__init__(data, uid) def user_arg(self): @@ -380,6 +380,8 @@ class User(SystemObject): args = ['--uid', self.id] if self.tenant: args += ['--tenant', self.tenant] + if self.account: + args += ['--account-id', self.account, '--account-root'] return args def build_command(self, command): diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 70e003c050c0e..f0b36865ed1c5 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -545,10 +545,10 @@ def create_role_per_zone(zonegroup_conns, roles_per_zone = 1): for i in range(roles_per_zone): role_name = gen_role_name() log.info('create role zone=%s name=%s', zone.name, role_name) - policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}" - role = zone.create_role("", role_name, policy_document, "") + policy_document = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]}) + zone.iam_conn.create_role(RoleName=role_name, AssumeRolePolicyDocument=policy_document) roles.append(role_name) - zone_role.append((zone, role)) + zone_role.append((zone, role_name)) return roles, zone_role @@ -639,9 +639,131 @@ def check_bucket_eq(zone_conn1, zone_conn2, bucket): if zone_conn2.zone.has_buckets(): zone_conn2.check_bucket_eq(zone_conn1, bucket.name) -def check_role_eq(zone_conn1, zone_conn2, role): - if zone_conn2.zone.has_roles(): - zone_conn2.check_role_eq(zone_conn1, role['create_role_response']['create_role_result']['role']['role_name']) +def check_role_eq(zone_conn1, zone_conn2, role_name): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + r1 = iam1.get_role(RoleName=role_name) + r2 = iam2.get_role(RoleName=role_name) + eq(r1['Role'], r2['Role']) + + # compare inline policies + policies1 = iam1.get_paginator('list_role_policies').paginate(RoleName=role_name) + policies2 = iam2.get_paginator('list_role_policies').paginate(RoleName=role_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['PolicyNames'], p2['PolicyNames']) + + # compare managed policies + policies1 = iam1.get_paginator('list_attached_role_policies').paginate(RoleName=role_name) + policies2 = iam2.get_paginator('list_attached_role_policies').paginate(RoleName=role_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['AttachedPolicies'], p2['AttachedPolicies']) + +def check_roles_eq(zone_conn1, zone_conn2): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + roles1 = iam1.get_paginator('list_roles').paginate() + roles2 = iam2.get_paginator('list_roles').paginate() + for r1, r2 in zip(roles1, roles2): + eq(r1['Roles'], r2['Roles']) + + for role in r1['Roles']: + check_role_eq(zone_conn1, zone_conn2, role['RoleName']) + +def check_user_eq(zone_conn1, zone_conn2, user_name): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + r1 = iam1.get_user(UserName=user_name) + r2 = iam2.get_user(UserName=user_name) + eq(r1['User'], r2['User']) + + # compare access keys + keys1 = iam1.get_paginator('list_access_keys').paginate(UserName=user_name) + keys2 = iam2.get_paginator('list_access_keys').paginate(UserName=user_name) + for k1, k2 in zip(keys1, keys2): + eq(k1['AccessKeyMetadata'], k2['AccessKeyMetadata']) + + # compare group memberships + groups1 = iam1.get_paginator('list_groups_for_user').paginate(UserName=user_name) + groups2 = iam2.get_paginator('list_groups_for_user').paginate(UserName=user_name) + for g1, g2 in zip(groups1, groups2): + eq(g1['Groups'], g2['Groups']) + + # compare inline policies + policies1 = iam1.get_paginator('list_user_policies').paginate(UserName=user_name) + policies2 = iam2.get_paginator('list_user_policies').paginate(UserName=user_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['PolicyNames'], p2['PolicyNames']) + + # compare managed policies + policies1 = iam1.get_paginator('list_attached_user_policies').paginate(UserName=user_name) + policies2 = iam2.get_paginator('list_attached_user_policies').paginate(UserName=user_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['AttachedPolicies'], p2['AttachedPolicies']) + +def check_users_eq(zone_conn1, zone_conn2): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + users1 = iam1.get_paginator('list_users').paginate() + users2 = iam2.get_paginator('list_users').paginate() + for u1, u2 in zip(users1, users2): + eq(u1['Users'], u2['Users']) + + for user in u1['Users']: + check_user_eq(zone_conn1, zone_conn2, user['UserName']) + +def check_group_eq(zone_conn1, zone_conn2, group_name): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + r1 = iam1.get_group(GroupName=group_name) + r2 = iam2.get_group(GroupName=group_name) + eq(r1['Group'], r2['Group']) + + # compare inline policies + policies1 = iam1.get_paginator('list_group_policies').paginate(GroupName=group_name) + policies2 = iam2.get_paginator('list_group_policies').paginate(GroupName=group_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['PolicyNames'], p2['PolicyNames']) + + # compare managed policies + policies1 = iam1.get_paginator('list_attached_group_policies').paginate(GroupName=group_name) + policies2 = iam2.get_paginator('list_attached_group_policies').paginate(GroupName=group_name) + for p1, p2 in zip(policies1, policies2): + eq(p1['AttachedPolicies'], p2['AttachedPolicies']) + +def check_groups_eq(zone_conn1, zone_conn2): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + groups1 = iam1.get_paginator('list_groups').paginate() + groups2 = iam2.get_paginator('list_groups').paginate() + for g1, g2 in zip(groups1, groups2): + eq(g1['Groups'], g2['Groups']) + + for group in g1['Groups']: + check_group_eq(zone_conn1, zone_conn2, group['GroupName']) + +def check_oidc_provider_eq(zone_conn1, zone_conn2, arn): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + p1 = iam1.get_open_id_connect_provider(OpenIDConnectProviderArn=arn) + p2 = iam2.get_open_id_connect_provider(OpenIDConnectProviderArn=arn) + eq(p1, p2) + +def check_oidc_providers_eq(zone_conn1, zone_conn2): + iam1 = zone_conn1.iam_conn + iam2 = zone_conn2.iam_conn + + providers1 = iam1.list_open_id_connect_providers()['OpenIDConnectProviderList'] + providers2 = iam2.list_open_id_connect_providers()['OpenIDConnectProviderList'] + for p1, p2 in zip(providers1, providers2): + eq(p1, p2) + check_oidc_provider_eq(zone_conn1, zone_conn2, p1['Arn']) def test_object_sync(): zonegroup = realm.master_zonegroup() @@ -1770,34 +1892,33 @@ def test_role_sync(): zonegroup_meta_checkpoint(zonegroup) - for source_conn, role in zone_role: - for target_conn in zonegroup_conns.zones: - if source_conn.zone == target_conn.zone: - continue - - check_role_eq(source_conn, target_conn, role) + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) def test_role_delete_sync(): zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) role_name = gen_role_name() log.info('create role zone=%s name=%s', zonegroup_conns.master_zone.name, role_name) - zonegroup_conns.master_zone.create_role("", role_name, None, "") + policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}" + zonegroup_conns.master_zone.iam_conn.create_role(RoleName=role_name, AssumeRolePolicyDocument=policy_document) zonegroup_meta_checkpoint(zonegroup) for zone in zonegroup_conns.zones: log.info(f'checking if zone: {zone.name} has role: {role_name}') - assert(zone.has_role(role_name)) + zone.iam_conn.get_role(RoleName=role_name) log.info(f'success, zone: {zone.name} has role: {role_name}') log.info(f"deleting role: {role_name}") - zonegroup_conns.master_zone.delete_role(role_name) + zonegroup_conns.master_zone.iam_conn.delete_role(RoleName=role_name) zonegroup_meta_checkpoint(zonegroup) for zone in zonegroup_conns.zones: log.info(f'checking if zone: {zone.name} does not have role: {role_name}') - assert(not zone.has_role(role_name)) + assert_raises(zone.iam_conn.exceptions.NoSuchEntityException, + zone.iam_conn.get_role, RoleName=role_name) log.info(f'success, zone: {zone.name} does not have role: {role_name}') @@ -3248,3 +3369,71 @@ def test_topic_notification_sync(): for conn in zonegroup_conns.zones: topic_list = conn.list_topics() assert_equal(len(topic_list), 0) + +def test_account_metadata_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) + managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + # create user, add access key, user policy, managed policy + iam.create_user(UserName=name) + iam.create_access_key(UserName=name) + iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + # create group, group policy, managed policy, add user to group + iam.create_group(GroupName=name) + iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.add_user_to_group(GroupName=name, UserName=name) + # create role, role policy, managed policy + iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) + iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + # TODO: test oidc provider + #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + + #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') + + iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + iam.delete_role_policy(RoleName=name, PolicyName='Allow') + iam.delete_role(RoleName=name) + + iam.remove_user_from_group(GroupName=name, UserName=name) + iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.delete_group_policy(GroupName=name, PolicyName='Allow') + iam.delete_group(GroupName=name) + + iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + iam.delete_user_policy(UserName=name, PolicyName='Allow') + key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] + iam.delete_access_key(UserName=name, AccessKeyId=key_id) + iam.delete_user(UserName=name) + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) diff --git a/src/test/rgw/test_multi.py b/src/test/rgw/test_multi.py index 57d27343efc33..cef6850c88a03 100644 --- a/src/test/rgw/test_multi.py +++ b/src/test/rgw/test_multi.py @@ -246,7 +246,7 @@ def init(parse_args): admin_user = multisite.User('zone.user') user_creds = gen_credentials() - user = multisite.User('tester', tenant=args.tenant) + user = multisite.User('tester', tenant=args.tenant, account='RGW11111111111111111') realm = multisite.Realm('r') if bootstrap: @@ -381,8 +381,9 @@ def init(parse_args): arg = ['--display-name', '"Zone User"', '--system'] arg += admin_creds.credential_args() admin_user.create(zone, arg) - # create test user - arg = ['--display-name', '"Test User"', '--caps', 'roles=*'] + # create test account/user + cluster.admin(['account', 'create', '--account-id', user.account]) + arg = ['--display-name', 'TestUser'] arg += user_creds.credential_args() user.create(zone, arg) else: -- 2.39.5