--- /dev/null
+"""
+Run a set of s3vectors tests on rgw.
+"""
+from io import BytesIO
+from configobj import ConfigObj
+import base64
+import contextlib
+import logging
+import os
+import random
+import string
+
+from teuthology import misc as teuthology
+from teuthology import contextutil
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def download(ctx, config):
+ assert isinstance(config, dict)
+ log.info('Downloading s3vectors-tests...')
+ testdir = teuthology.get_testdir(ctx)
+ branch = ctx.config.get('suite_branch')
+ repo = ctx.config.get('suite_repo')
+ log.info('Using branch %s from %s for s3vectors tests', branch, repo)
+ for (client, client_config) in config.items():
+
+ ctx.cluster.only(client).run(
+ args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)],
+ )
+
+ sha1 = client_config.get('sha1')
+
+ if sha1 is not None:
+ ctx.cluster.only(client).run(
+ args=[
+ 'cd', '{tdir}/ceph'.format(tdir=testdir),
+ run.Raw('&&'),
+ 'git', 'reset', '--hard', sha1,
+ ],
+ )
+
+ try:
+ yield
+ finally:
+ log.info('Removing s3vectors-tests...')
+ testdir = teuthology.get_testdir(ctx)
+ for client in config:
+ ctx.cluster.only(client).run(
+ args=[
+ 'rm',
+ '-rf',
+ '{tdir}/ceph'.format(tdir=testdir),
+ ],
+ )
+
+
+def _config_user(s3vtests_conf, section, user):
+ """
+ Configure users for this section by stashing away keys, ids, and
+ email addresses.
+ """
+ s3vtests_conf[section].setdefault('user_id', user)
+ s3vtests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
+ s3vtests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
+ s3vtests_conf[section].setdefault('access_key',
+ ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
+ s3vtests_conf[section].setdefault('secret_key',
+ base64.b64encode(os.urandom(40)).decode())
+
+
+@contextlib.contextmanager
+def create_users(ctx, config):
+ """
+ Create a main and an alternate s3 user.
+ """
+ assert isinstance(config, dict)
+ log.info('Creating rgw user...')
+ testdir = teuthology.get_testdir(ctx)
+
+ users = {'s3 main': 'foo'}
+ for client in config['clients']:
+ s3vtests_conf = config['s3vtests_conf'][client]
+ for section, user in users.items():
+ _config_user(s3vtests_conf, section, '{user}.{client}'.format(user=user, client=client))
+ log.debug('Creating user {user} on {host}'.format(user=s3vtests_conf[section]['user_id'], host=client))
+ cluster_name, daemon_type, client_id = teuthology.split_role(client)
+ client_with_id = daemon_type + '.' + client_id
+ ctx.cluster.only(client).run(
+ args=[
+ 'adjust-ulimits',
+ 'ceph-coverage',
+ '{tdir}/archive/coverage'.format(tdir=testdir),
+ 'radosgw-admin',
+ '-n', client_with_id,
+ 'user', 'create',
+ '--uid', s3vtests_conf[section]['user_id'],
+ '--display-name', s3vtests_conf[section]['display_name'],
+ '--access-key', s3vtests_conf[section]['access_key'],
+ '--secret', s3vtests_conf[section]['secret_key'],
+ '--cluster', cluster_name,
+ ],
+ )
+
+ try:
+ yield
+ finally:
+ for client in config['clients']:
+ for user in users.values():
+ uid = '{user}.{client}'.format(user=user, client=client)
+ cluster_name, daemon_type, client_id = teuthology.split_role(client)
+ client_with_id = daemon_type + '.' + client_id
+ ctx.cluster.only(client).run(
+ args=[
+ 'adjust-ulimits',
+ 'ceph-coverage',
+ '{tdir}/archive/coverage'.format(tdir=testdir),
+ 'radosgw-admin',
+ '-n', client_with_id,
+ 'user', 'rm',
+ '--uid', uid,
+ '--purge-data',
+ '--cluster', cluster_name,
+ ],
+ )
+
+
+@contextlib.contextmanager
+def configure(ctx, config):
+ assert isinstance(config, dict)
+ log.info('Configuring s3vectors-tests...')
+ testdir = teuthology.get_testdir(ctx)
+ for client, properties in config['clients'].items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ s3vtests_conf = config['s3vtests_conf'][client]
+
+ conf_fp = BytesIO()
+ s3vtests_conf.write(conf_fp)
+ remote.write_file(
+ path='{tdir}/ceph/src/test/rgw/s3vectors/s3vectors-tests.{client}.conf'.format(tdir=testdir, client=client),
+ data=conf_fp.getvalue(),
+ )
+
+ try:
+ yield
+ finally:
+ log.info('Removing s3vectors-tests.conf file...')
+ testdir = teuthology.get_testdir(ctx)
+ for client, properties in config['clients'].items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ remote.run(
+ args=['rm', '-f',
+ '{tdir}/ceph/src/test/rgw/s3vectors/s3vectors-tests.{client}.conf'.format(tdir=testdir,client=client),
+ ],
+ )
+
+
+def get_toxvenv_dir(ctx):
+ return ctx.tox.venv_path
+
+
+def toxvenv_sh(ctx, remote, args, **kwargs):
+ activate = get_toxvenv_dir(ctx) + '/bin/activate'
+ return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)
+
+
+@contextlib.contextmanager
+def run_tests(ctx, config):
+ """
+ Run the s3vectors tests after everything is set up.
+ :param ctx: Context passed to task
+ :param config: specific configuration information
+ """
+ assert isinstance(config, dict)
+ log.info('Running s3vectors-tests...')
+ testdir = teuthology.get_testdir(ctx)
+ for client, client_config in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+
+ # test marks to use by default
+ attr = ['vector_bucket_test', 'index_test', 'vector_test']
+
+ if 'extra_attr' in client_config:
+ attr = client_config.get('extra_attr')
+
+ args = ['cd', '{tdir}/ceph/src/test/rgw/s3vectors/'.format(tdir=testdir), run.Raw('&&'),
+ 'S3VTESTS_CONF=./s3vectors-tests.{client}.conf'.format(client=client),
+ 'tox', '--', '-v', '-m', ' or '.join(attr)]
+
+ toxvenv_sh(ctx, remote, args, label="s3vectors tests against rgw")
+
+ yield
+
+
+@contextlib.contextmanager
+def task(ctx,config):
+ """
+ If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo'
+ parameters in your teuthology-suite command. Example command for this is as follows::
+
+ teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:s3vectors --ceph your_ceph_branch_name --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name
+ """
+ assert hasattr(ctx, 'rgw'), 's3vtests must run after the rgw task'
+ assert hasattr(ctx, 'tox'), 's3vtests must run after the tox task'
+ assert config is None or isinstance(config, list) \
+ or isinstance(config, dict), \
+ "task only supports a list or dictionary for configuration"
+
+ all_clients = ['client.{id}'.format(id=id_)
+ for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+ if config is None:
+ config = all_clients
+ if isinstance(config, list):
+ config = dict.fromkeys(config)
+ clients=config.keys()
+
+ log.debug('config is %s', config)
+
+ s3vtests_conf = {}
+
+ for client in clients:
+ endpoint = ctx.rgw.role_endpoints.get(client)
+ assert endpoint, 's3vtests: no rgw endpoint for {}'.format(client)
+
+ s3vtests_conf[client] = ConfigObj(
+ indent_type='',
+ infile={
+ 'DEFAULT':
+ {
+ 'port':endpoint.port,
+ 'host':endpoint.dns_name,
+ 'zonegroup':ctx.rgw.zonegroup,
+ },
+ 's3 main':{}
+ }
+ )
+
+ with contextutil.nested(
+ lambda: download(ctx=ctx, config=config),
+ lambda: create_users(ctx=ctx, config=dict(
+ clients=clients,
+ s3vtests_conf=s3vtests_conf,
+ )),
+ lambda: configure(ctx=ctx, config=dict(
+ clients=config,
+ s3vtests_conf=s3vtests_conf,
+ )),
+ lambda: run_tests(ctx=ctx, config=config),
+ ):
+ pass
+ yield
+
projects / ceph.git / commitdiff