From 895121fb3d319e0400efd34f01ab5d557b151877 Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Wed, 23 Apr 2025 09:00:45 +0000 Subject: [PATCH] test/rgw/dedup: add dedup test suite to teuthology Signed-off-by: Yuval Lifshitz --- qa/suites/rgw/dedup/% | 0 qa/suites/rgw/dedup/.qa | 1 + qa/suites/rgw/dedup/beast.yaml | 1 + qa/suites/rgw/dedup/bluestore-bitmap.yaml | 1 + qa/suites/rgw/dedup/fixed-3-rgw.yaml | 13 + .../rgw/dedup/ignore-pg-availability.yaml | 1 + qa/suites/rgw/dedup/overrides.yaml | 12 + qa/suites/rgw/dedup/supported-distros | 1 + qa/suites/rgw/dedup/tasks/+ | 0 qa/suites/rgw/dedup/tasks/.qa | 1 + qa/suites/rgw/dedup/tasks/0-install.yaml | 13 + qa/suites/rgw/dedup/tasks/test_dedup.yaml | 5 + qa/tasks/dedup_tests.py | 254 ++++++++++++++++++ 13 files changed, 303 insertions(+) create mode 100644 qa/suites/rgw/dedup/% create mode 120000 qa/suites/rgw/dedup/.qa create mode 120000 qa/suites/rgw/dedup/beast.yaml create mode 120000 qa/suites/rgw/dedup/bluestore-bitmap.yaml create mode 100644 qa/suites/rgw/dedup/fixed-3-rgw.yaml create mode 120000 qa/suites/rgw/dedup/ignore-pg-availability.yaml create mode 100644 qa/suites/rgw/dedup/overrides.yaml create mode 120000 qa/suites/rgw/dedup/supported-distros create mode 100644 qa/suites/rgw/dedup/tasks/+ create mode 120000 qa/suites/rgw/dedup/tasks/.qa create mode 100644 qa/suites/rgw/dedup/tasks/0-install.yaml create mode 100644 qa/suites/rgw/dedup/tasks/test_dedup.yaml create mode 100644 qa/tasks/dedup_tests.py diff --git a/qa/suites/rgw/dedup/% b/qa/suites/rgw/dedup/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/rgw/dedup/.qa b/qa/suites/rgw/dedup/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/rgw/dedup/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rgw/dedup/beast.yaml b/qa/suites/rgw/dedup/beast.yaml new file mode 120000 index 0000000000000..09ced62c42aa8 --- /dev/null +++ b/qa/suites/rgw/dedup/beast.yaml @@ -0,0 +1 @@ +.qa/rgw_frontend/beast.yaml \ No newline at end of file diff --git a/qa/suites/rgw/dedup/bluestore-bitmap.yaml b/qa/suites/rgw/dedup/bluestore-bitmap.yaml new file mode 120000 index 0000000000000..a59cf5175069a --- /dev/null +++ b/qa/suites/rgw/dedup/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/objectstore/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/rgw/dedup/fixed-3-rgw.yaml b/qa/suites/rgw/dedup/fixed-3-rgw.yaml new file mode 100644 index 0000000000000..fc86b982d2e2a --- /dev/null +++ b/qa/suites/rgw/dedup/fixed-3-rgw.yaml @@ -0,0 +1,13 @@ +roles: +- [mon.a, mon.c, mgr.y, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mgr.x, osd.4, osd.5, osd.6, osd.7, client.1] +- [client.2] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/qa/suites/rgw/dedup/ignore-pg-availability.yaml b/qa/suites/rgw/dedup/ignore-pg-availability.yaml new file mode 120000 index 0000000000000..32340b1fa8be8 --- /dev/null +++ b/qa/suites/rgw/dedup/ignore-pg-availability.yaml @@ -0,0 +1 @@ +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/qa/suites/rgw/dedup/overrides.yaml b/qa/suites/rgw/dedup/overrides.yaml new file mode 100644 index 0000000000000..a36e3465dadc7 --- /dev/null +++ b/qa/suites/rgw/dedup/overrides.yaml @@ -0,0 +1,12 @@ +overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph + debug rgw: 20 + debug rgw dedup: 20 + rgw: + storage classes: + LUKEWARM: + FROZEN: diff --git a/qa/suites/rgw/dedup/supported-distros b/qa/suites/rgw/dedup/supported-distros new file mode 120000 index 0000000000000..78f2991b407af --- /dev/null +++ b/qa/suites/rgw/dedup/supported-distros @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$/ \ No newline at end of file diff --git a/qa/suites/rgw/dedup/tasks/+ b/qa/suites/rgw/dedup/tasks/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/rgw/dedup/tasks/.qa b/qa/suites/rgw/dedup/tasks/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/rgw/dedup/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rgw/dedup/tasks/0-install.yaml b/qa/suites/rgw/dedup/tasks/0-install.yaml new file mode 100644 index 0000000000000..b60e932ab0e3d --- /dev/null +++ b/qa/suites/rgw/dedup/tasks/0-install.yaml @@ -0,0 +1,13 @@ +tasks: +- install: +- ceph: +- openssl_keys: +- rgw: [client.0, client.1, client.2] +- tox: [client.0] + +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 10 + osd_max_pg_log_entries: 10 diff --git a/qa/suites/rgw/dedup/tasks/test_dedup.yaml b/qa/suites/rgw/dedup/tasks/test_dedup.yaml new file mode 100644 index 0000000000000..d6ecca18ffe7f --- /dev/null +++ b/qa/suites/rgw/dedup/tasks/test_dedup.yaml @@ -0,0 +1,5 @@ +tasks: +- tox: [client.0] +- dedup-tests: + client.0: + rgw_server: client.0 diff --git a/qa/tasks/dedup_tests.py b/qa/tasks/dedup_tests.py new file mode 100644 index 0000000000000..a801fcbc24596 --- /dev/null +++ b/qa/tasks/dedup_tests.py @@ -0,0 +1,254 @@ +""" +Run a set of dedup tests on rgw. +""" +from io import BytesIO +from configobj import ConfigObj +import base64 +import contextlib +import logging +import os +import random +import string + +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.orchestra import run + +log = logging.getLogger(__name__) + +@contextlib.contextmanager +def download(ctx, config): + assert isinstance(config, dict) + log.info('Downloading dedup-tests...') + testdir = teuthology.get_testdir(ctx) + branch = ctx.config.get('suite_branch') + repo = ctx.config.get('suite_repo') + log.info('Using branch %s from %s for dedup tests', branch, repo) + for (client, client_config) in config.items(): + + ctx.cluster.only(client).run( + args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)], + ) + + sha1 = client_config.get('sha1') + + if sha1 is not None: + ctx.cluster.only(client).run( + args=[ + 'cd', '{tdir}/ceph'.format(tdir=testdir), + run.Raw('&&'), + 'git', 'reset', '--hard', sha1, + ], + ) + + try: + yield + finally: + log.info('Removing dedup-tests...') + testdir = teuthology.get_testdir(ctx) + for client in config: + ctx.cluster.only(client).run( + args=[ + 'rm', + '-rf', + '{tdir}/ceph'.format(tdir=testdir), + ], + ) + + +def _config_user(deduptests_conf, section, user): + """ + Configure users for this section by stashing away keys, ids, and + email addresses. + """ + deduptests_conf[section].setdefault('user_id', user) + deduptests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user)) + deduptests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user)) + deduptests_conf[section].setdefault('access_key', + ''.join(random.choice(string.ascii_uppercase) for i in range(20))) + deduptests_conf[section].setdefault('secret_key', + base64.b64encode(os.urandom(40)).decode()) + + +@contextlib.contextmanager +def create_users(ctx, config): + """ + Create a main and an alternate s3 user. + """ + assert isinstance(config, dict) + log.info('Creating rgw user...') + testdir = teuthology.get_testdir(ctx) + + users = {'s3 main': 'foo'} + for client in config['clients']: + deduptests_conf = config['deduptests_conf'][client] + for section, user in users.items(): + _config_user(deduptests_conf, section, '{user}.{client}'.format(user=user, client=client)) + log.debug('Creating user {user} on {host}'.format(user=deduptests_conf[section]['user_id'], host=client)) + cluster_name, daemon_type, client_id = teuthology.split_role(client) + client_with_id = daemon_type + '.' + client_id + ctx.cluster.only(client).run( + args=[ + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=testdir), + 'radosgw-admin', + '-n', client_with_id, + 'user', 'create', + '--uid', deduptests_conf[section]['user_id'], + '--display-name', deduptests_conf[section]['display_name'], + '--access-key', deduptests_conf[section]['access_key'], + '--secret', deduptests_conf[section]['secret_key'], + '--cluster', cluster_name, + ], + ) + + try: + yield + finally: + for client in config['clients']: + for user in users.values(): + uid = '{user}.{client}'.format(user=user, client=client) + cluster_name, daemon_type, client_id = teuthology.split_role(client) + client_with_id = daemon_type + '.' + client_id + ctx.cluster.only(client).run( + args=[ + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=testdir), + 'radosgw-admin', + '-n', client_with_id, + 'user', 'rm', + '--uid', uid, + '--purge-data', + '--cluster', cluster_name, + ], + ) + + +@contextlib.contextmanager +def configure(ctx, config): + assert isinstance(config, dict) + log.info('Configuring dedup-tests...') + testdir = teuthology.get_testdir(ctx) + for client, properties in config['clients'].items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + deduptests_conf = config['deduptests_conf'][client] + + conf_fp = BytesIO() + deduptests_conf.write(conf_fp) + remote.write_file( + path='{tdir}/ceph/src/test/rgw/dedup/deduptests.{client}.conf'.format(tdir=testdir, client=client), + data=conf_fp.getvalue(), + ) + + try: + yield + finally: + log.info('Removing dedup-tests.conf file...') + testdir = teuthology.get_testdir(ctx) + for client, properties in config['clients'].items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + remote.run( + args=['rm', '-f', + '{tdir}/ceph/src/test/rgw/dedup/deduptests.{client}.conf'.format(tdir=testdir,client=client), + ], + ) + + +def get_toxvenv_dir(ctx): + return ctx.tox.venv_path + + +def toxvenv_sh(ctx, remote, args, **kwargs): + activate = get_toxvenv_dir(ctx) + '/bin/activate' + return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs) + + +@contextlib.contextmanager +def run_tests(ctx, config): + """ + Run the dedup tests after everything is set up. + :param ctx: Context passed to task + :param config: specific configuration information + """ + assert isinstance(config, dict) + log.info('Running dedup-tests...') + testdir = teuthology.get_testdir(ctx) + for client, client_config in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + + # test marks to use by default + attr = ['basic_test', 'request_test', 'example_test'] + + if 'extra_attr' in client_config: + attr = client_config.get('extra_attr') + + args = ['cd', '{tdir}/ceph/src/test/rgw/dedup/'.format(tdir=testdir), run.Raw('&&'), + 'DEDUPTESTS_CONF=./deduptests.{client}.conf'.format(client=client), + 'tox', '--', '-v', '-m', ' or '.join(attr)] + + toxvenv_sh(ctx, remote, args, label="dedup tests against rgw") + + yield + + +@contextlib.contextmanager +def task(ctx,config): + """ + + If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo' + parameters in your teuthology-suite command. Example command for this is as follows:: + + teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:dedup --ceph your_ceph_branch_name -m smithi --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name + + """ + assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task' + assert hasattr(ctx, 'tox'), 's3tests must run after the tox task' + assert config is None or isinstance(config, list) \ + or isinstance(config, dict), \ + "task only supports a list or dictionary for configuration" + + all_clients = ['client.{id}'.format(id=id_) + for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] + if config is None: + config = all_clients + if isinstance(config, list): + config = dict.fromkeys(config) + clients=config.keys() + + log.debug('config is %s', config) + + deduptests_conf = {} + + for client in clients: + endpoint = ctx.rgw.role_endpoints.get(client) + assert endpoint, 'deduptests: no rgw endpoint for {}'.format(client) + + deduptests_conf[client] = ConfigObj( + indent_type='', + infile={ + 'DEFAULT': + { + 'port':endpoint.port, + 'host':endpoint.dns_name, + }, + 's3 main':{} + } + ) + + with contextutil.nested( + lambda: download(ctx=ctx, config=config), + lambda: create_users(ctx=ctx, config=dict( + clients=clients, + deduptests_conf=deduptests_conf, + )), + lambda: configure(ctx=ctx, config=dict( + clients=config, + deduptests_conf=deduptests_conf, + )), + lambda: run_tests(ctx=ctx, config=config), + ): + pass + yield + -- 2.39.5