From: Yuval Lifshitz Date: Wed, 4 Feb 2026 17:57:21 +0000 (+0000) Subject: tests/rgw/s3vector: add s3vector tests to teuthology X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0c786ddb6b5e6827a6400068ec39b79ad8e7f605;p=ceph.git tests/rgw/s3vector: add s3vector tests to teuthology also fix shaman build failures with lancedb-c. see: https://shaman.ceph.com/builds/ceph/wip-add-lancedb/e7ce95d8e2e279b373f3c486bd1f64c39b23883f/ note that currently we are testing only against centos since the rust version in ubuntu jammy and noble is too old for lancedb liblancedb.so is installed as part of the ceph_base Signed-off-by: Yuval Lifshitz --- diff --git a/ceph.spec.in b/ceph.spec.in index 9d3fdc66d38..e2f9eb253e9 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -151,6 +151,13 @@ %bcond_with pypkg %endif +%if 0%{with lancedb} +# liblancedb.so is a bundled Rust library without a SONAME, so RPM's +# auto-provides won't detect it. Exclude it from auto-requires to avoid +# an unsatisfiable dependency. +%global __requires_exclude ^liblancedb\\.so.*$ +%endif + %{!?_udevrulesdir: %global _udevrulesdir /lib/udev/rules.d} %{!?tmpfiles_create: %global tmpfiles_create systemd-tmpfiles --create} %{!?python3_pkgversion: %global python3_pkgversion 3} @@ -2090,6 +2097,9 @@ rm -rf %{_vpath_builddir} %{_libdir}/libosd_tp.so* %{_libdir}/libmgr_op_tp.so* %endif +%if 0%{with lancedb} +%{_libdir}/liblancedb.so +%endif %config(noreplace) %{_sysconfdir}/logrotate.d/ceph %if 0%{?fedora} || 0%{?rhel} || 0%{?openEuler} %config(noreplace) %{_sysconfdir}/sysconfig/ceph diff --git a/debian/control b/debian/control index bc03b68e8fd..074285a4fce 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,7 @@ Uploaders: Ken Dreyer , Alfredo Deza , Build-Depends: automake, bison, - cargo, + cargo (>= 1.78), cmake (>= 3.10.2), cpio, cython3, diff --git a/qa/suites/rgw/s3vectors/% b/qa/suites/rgw/s3vectors/% new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rgw/s3vectors/.qa b/qa/suites/rgw/s3vectors/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rgw/s3vectors/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/beast.yaml b/qa/suites/rgw/s3vectors/beast.yaml new file mode 120000 index 00000000000..09ced62c42a --- /dev/null +++ b/qa/suites/rgw/s3vectors/beast.yaml @@ -0,0 +1 @@ +.qa/rgw_frontend/beast.yaml \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/bluestore-bitmap.yaml b/qa/suites/rgw/s3vectors/bluestore-bitmap.yaml new file mode 120000 index 00000000000..a59cf517506 --- /dev/null +++ b/qa/suites/rgw/s3vectors/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/objectstore/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/centos_latest.yaml b/qa/suites/rgw/s3vectors/centos_latest.yaml new file mode 120000 index 00000000000..bd9854e7029 --- /dev/null +++ b/qa/suites/rgw/s3vectors/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/fixed-1.yaml b/qa/suites/rgw/s3vectors/fixed-1.yaml new file mode 120000 index 00000000000..02df5dd0cd0 --- /dev/null +++ b/qa/suites/rgw/s3vectors/fixed-1.yaml @@ -0,0 +1 @@ +.qa/clusters/fixed-1.yaml \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/ignore-pg-availability.yaml b/qa/suites/rgw/s3vectors/ignore-pg-availability.yaml new file mode 120000 index 00000000000..32340b1fa8b --- /dev/null +++ b/qa/suites/rgw/s3vectors/ignore-pg-availability.yaml @@ -0,0 +1 @@ +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/overrides.yaml b/qa/suites/rgw/s3vectors/overrides.yaml new file mode 100644 index 00000000000..ae4da0b17b4 --- /dev/null +++ b/qa/suites/rgw/s3vectors/overrides.yaml @@ -0,0 +1,11 @@ +overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph + debug rgw: 20 + rgw: + storage classes: + LUKEWARM: + FROZEN: diff --git a/qa/suites/rgw/s3vectors/tasks/+ b/qa/suites/rgw/s3vectors/tasks/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rgw/s3vectors/tasks/.qa b/qa/suites/rgw/s3vectors/tasks/.qa new file mode 120000 index 00000000000..a602a0353e7 --- /dev/null +++ b/qa/suites/rgw/s3vectors/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rgw/s3vectors/tasks/0-install.yaml b/qa/suites/rgw/s3vectors/tasks/0-install.yaml new file mode 100644 index 00000000000..6cf82f57476 --- /dev/null +++ b/qa/suites/rgw/s3vectors/tasks/0-install.yaml @@ -0,0 +1,13 @@ +tasks: +- install: +- ceph: +- openssl_keys: +- rgw: [client.0] +- tox: [client.0] + +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 10 + osd_max_pg_log_entries: 10 diff --git a/qa/suites/rgw/s3vectors/tasks/test_s3vectors.yaml b/qa/suites/rgw/s3vectors/tasks/test_s3vectors.yaml new file mode 100644 index 00000000000..80e998b15a1 --- /dev/null +++ b/qa/suites/rgw/s3vectors/tasks/test_s3vectors.yaml @@ -0,0 +1,5 @@ +tasks: +- tox: [client.0] +- s3vectors-tests: + client.0: + rgw_server: client.0 diff --git a/qa/tasks/s3vectors_tests.py b/qa/tasks/s3vectors_tests.py new file mode 100644 index 00000000000..877205530da --- /dev/null +++ b/qa/tasks/s3vectors_tests.py @@ -0,0 +1,253 @@ +""" +Run a set of s3vectors tests on rgw. +""" +from io import BytesIO +from configobj import ConfigObj +import base64 +import contextlib +import logging +import os +import random +import string + +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.orchestra import run + +log = logging.getLogger(__name__) + +@contextlib.contextmanager +def download(ctx, config): + assert isinstance(config, dict) + log.info('Downloading s3vectors-tests...') + testdir = teuthology.get_testdir(ctx) + branch = ctx.config.get('suite_branch') + repo = ctx.config.get('suite_repo') + log.info('Using branch %s from %s for s3vectors tests', branch, repo) + for (client, client_config) in config.items(): + + ctx.cluster.only(client).run( + args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)], + ) + + sha1 = client_config.get('sha1') + + if sha1 is not None: + ctx.cluster.only(client).run( + args=[ + 'cd', '{tdir}/ceph'.format(tdir=testdir), + run.Raw('&&'), + 'git', 'reset', '--hard', sha1, + ], + ) + + try: + yield + finally: + log.info('Removing s3vectors-tests...') + testdir = teuthology.get_testdir(ctx) + for client in config: + ctx.cluster.only(client).run( + args=[ + 'rm', + '-rf', + '{tdir}/ceph'.format(tdir=testdir), + ], + ) + + +def _config_user(s3vtests_conf, section, user): + """ + Configure users for this section by stashing away keys, ids, and + email addresses. + """ + s3vtests_conf[section].setdefault('user_id', user) + s3vtests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user)) + s3vtests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user)) + s3vtests_conf[section].setdefault('access_key', + ''.join(random.choice(string.ascii_uppercase) for i in range(20))) + s3vtests_conf[section].setdefault('secret_key', + base64.b64encode(os.urandom(40)).decode()) + + +@contextlib.contextmanager +def create_users(ctx, config): + """ + Create a main and an alternate s3 user. + """ + assert isinstance(config, dict) + log.info('Creating rgw user...') + testdir = teuthology.get_testdir(ctx) + + users = {'s3 main': 'foo'} + for client in config['clients']: + s3vtests_conf = config['s3vtests_conf'][client] + for section, user in users.items(): + _config_user(s3vtests_conf, section, '{user}.{client}'.format(user=user, client=client)) + log.debug('Creating user {user} on {host}'.format(user=s3vtests_conf[section]['user_id'], host=client)) + cluster_name, daemon_type, client_id = teuthology.split_role(client) + client_with_id = daemon_type + '.' + client_id + ctx.cluster.only(client).run( + args=[ + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=testdir), + 'radosgw-admin', + '-n', client_with_id, + 'user', 'create', + '--uid', s3vtests_conf[section]['user_id'], + '--display-name', s3vtests_conf[section]['display_name'], + '--access-key', s3vtests_conf[section]['access_key'], + '--secret', s3vtests_conf[section]['secret_key'], + '--cluster', cluster_name, + ], + ) + + try: + yield + finally: + for client in config['clients']: + for user in users.values(): + uid = '{user}.{client}'.format(user=user, client=client) + cluster_name, daemon_type, client_id = teuthology.split_role(client) + client_with_id = daemon_type + '.' + client_id + ctx.cluster.only(client).run( + args=[ + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=testdir), + 'radosgw-admin', + '-n', client_with_id, + 'user', 'rm', + '--uid', uid, + '--purge-data', + '--cluster', cluster_name, + ], + ) + + +@contextlib.contextmanager +def configure(ctx, config): + assert isinstance(config, dict) + log.info('Configuring s3vectors-tests...') + testdir = teuthology.get_testdir(ctx) + for client, properties in config['clients'].items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + s3vtests_conf = config['s3vtests_conf'][client] + + conf_fp = BytesIO() + s3vtests_conf.write(conf_fp) + remote.write_file( + path='{tdir}/ceph/src/test/rgw/s3vectors/s3vectors-tests.{client}.conf'.format(tdir=testdir, client=client), + data=conf_fp.getvalue(), + ) + + try: + yield + finally: + log.info('Removing s3vectors-tests.conf file...') + testdir = teuthology.get_testdir(ctx) + for client, properties in config['clients'].items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + remote.run( + args=['rm', '-f', + '{tdir}/ceph/src/test/rgw/s3vectors/s3vectors-tests.{client}.conf'.format(tdir=testdir,client=client), + ], + ) + + +def get_toxvenv_dir(ctx): + return ctx.tox.venv_path + + +def toxvenv_sh(ctx, remote, args, **kwargs): + activate = get_toxvenv_dir(ctx) + '/bin/activate' + return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs) + + +@contextlib.contextmanager +def run_tests(ctx, config): + """ + Run the s3vectors tests after everything is set up. + :param ctx: Context passed to task + :param config: specific configuration information + """ + assert isinstance(config, dict) + log.info('Running s3vectors-tests...') + testdir = teuthology.get_testdir(ctx) + for client, client_config in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + + # test marks to use by default + attr = ['vector_bucket_test', 'index_test', 'vector_test'] + + if 'extra_attr' in client_config: + attr = client_config.get('extra_attr') + + args = ['cd', '{tdir}/ceph/src/test/rgw/s3vectors/'.format(tdir=testdir), run.Raw('&&'), + 'S3VTESTS_CONF=./s3vectors-tests.{client}.conf'.format(client=client), + 'tox', '--', '-v', '-m', ' or '.join(attr)] + + toxvenv_sh(ctx, remote, args, label="s3vectors tests against rgw") + + yield + + +@contextlib.contextmanager +def task(ctx,config): + """ + If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo' + parameters in your teuthology-suite command. Example command for this is as follows:: + + teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:s3vectors --ceph your_ceph_branch_name --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name + """ + assert hasattr(ctx, 'rgw'), 's3vtests must run after the rgw task' + assert hasattr(ctx, 'tox'), 's3vtests must run after the tox task' + assert config is None or isinstance(config, list) \ + or isinstance(config, dict), \ + "task only supports a list or dictionary for configuration" + + all_clients = ['client.{id}'.format(id=id_) + for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] + if config is None: + config = all_clients + if isinstance(config, list): + config = dict.fromkeys(config) + clients=config.keys() + + log.debug('config is %s', config) + + s3vtests_conf = {} + + for client in clients: + endpoint = ctx.rgw.role_endpoints.get(client) + assert endpoint, 's3vtests: no rgw endpoint for {}'.format(client) + + s3vtests_conf[client] = ConfigObj( + indent_type='', + infile={ + 'DEFAULT': + { + 'port':endpoint.port, + 'host':endpoint.dns_name, + 'zonegroup':ctx.rgw.zonegroup, + }, + 's3 main':{} + } + ) + + with contextutil.nested( + lambda: download(ctx=ctx, config=config), + lambda: create_users(ctx=ctx, config=dict( + clients=clients, + s3vtests_conf=s3vtests_conf, + )), + lambda: configure(ctx=ctx, config=dict( + clients=config, + s3vtests_conf=s3vtests_conf, + )), + lambda: run_tests(ctx=ctx, config=config), + ): + pass + yield + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 23da375960f..00df58517bb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1205,6 +1205,7 @@ if(WITH_RADOSGW) set(LANCEDB_BUILD_DIR "${CMAKE_BINARY_DIR}/src/lancedb-c-prefix/src/lancedb-c-build") # Determine the Rust target directory based on build type + set(LANCEDB_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lancedb-c-prefix/src/lancedb-c-build") if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(LANCEDB_RUST_TARGET_DIR "${LANCEDB_BUILD_DIR}/target/debug") else() diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 267b9978a03..98eb649f7f3 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -420,6 +420,8 @@ if(WITH_RADOSGW_LANCEDB) ) target_link_libraries(rgw_common PUBLIC lancedb) add_dependencies(rgw_common lancedb-c) + install(FILES ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/liblancedb.so + DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() set(rgw_a_srcs diff --git a/src/test/rgw/s3vectors/__init__.py b/src/test/rgw/s3vectors/__init__.py index 2ef443a267b..78224c16dbb 100644 --- a/src/test/rgw/s3vectors/__init__.py +++ b/src/test/rgw/s3vectors/__init__.py @@ -26,7 +26,11 @@ def setup(): global default_port default_port = int(defaults.get("port")) - # vars from the main section + + global default_zonegroup + default_zonegroup = defaults.get("zonegroup") + + # vars from the main section global main_access_key main_access_key = cfg.get('s3 main',"access_key") @@ -54,6 +58,11 @@ def get_config_port(): return default_port +def get_config_zonegroup(): + global default_zonegroup + return default_zonegroup + + def get_access_key(): global main_access_key return main_access_key diff --git a/src/test/rgw/s3vectors/s3vector_test.py b/src/test/rgw/s3vectors/s3vector_test.py index 97fef27557a..acfa9f1641c 100644 --- a/src/test/rgw/s3vectors/s3vector_test.py +++ b/src/test/rgw/s3vectors/s3vector_test.py @@ -19,6 +19,7 @@ from . import( configfile, get_config_host, get_config_port, + get_config_zonegroup, get_access_key, get_secret_key, get_config_host2, @@ -74,6 +75,7 @@ def connection(service_name='s3vectors'): endpoint_url=scheme+hostname+':'+str(port_no), aws_access_key_id=access_key, aws_secret_access_key=secret_key, + region_name=get_config_zonegroup(), config=config) return client @@ -103,6 +105,7 @@ def connection2(service_name='s3vectors'): endpoint_url=scheme+hostname+':'+str(port_no), aws_access_key_id=access_key, aws_secret_access_key=secret_key, + region_name=get_config_zonegroup(), config=config) return client @@ -129,6 +132,7 @@ def another_user(tenant=None): endpoint_url=scheme+hostname+':'+str(port_no), aws_access_key_id=access_key, aws_secret_access_key=secret_key, + region_name=get_config_zonegroup(), config=Config(signature_version='s3v4')) return client diff --git a/src/test/rgw/s3vectors/s3vtests.conf.SAMPLE b/src/test/rgw/s3vectors/s3vtests.conf.SAMPLE index eb3291dafa8..33994b002b3 100644 --- a/src/test/rgw/s3vectors/s3vtests.conf.SAMPLE +++ b/src/test/rgw/s3vectors/s3vtests.conf.SAMPLE @@ -1,6 +1,7 @@ [DEFAULT] port = 8000 host = localhost +zonegroup = default [s3 main] access_key = 0555b35654ad1656d804 diff --git a/src/test/rgw/s3vectors/s3vtests.conf.multisite b/src/test/rgw/s3vectors/s3vtests.conf.multisite index 3306db5b2ee..30c50d0a045 100644 --- a/src/test/rgw/s3vectors/s3vtests.conf.multisite +++ b/src/test/rgw/s3vectors/s3vtests.conf.multisite @@ -1,6 +1,7 @@ [DEFAULT] port = 8101 host = localhost +zonegroup = zg1 [secondary] port = 8201