From a50b342c769543a945d97abfc6054546cfd70493 Mon Sep 17 00:00:00 2001 From: ShreeJejurikar Date: Thu, 9 Apr 2026 14:18:24 +0530 Subject: [PATCH] rgw/test: add Journal mode support to bucket logging test suite Add --logging-type flag to run the Python bucket logging test suite in either Standard or Journal mode. The same tests run against both logging types with no changes to test logic or assertions. - Add --logging-type pytest CLI option (Standard default, Journal opt-in) - Detect boto3 LoggingType extension availability at session startup - Thread logging_type through helpers and test functions - Add teuthology task YAML for Journal mode suite runs - Install service-2.sdk-extras.json in the teuthology task when logging_type is Journal (s3tests cleans it up after its own run, so the file isn't available by the time our Journal job runs) - Document Journal mode local usage in the test suite README Signed-off-by: Shree Jejurikar --- .../tasks/test_bucket_logging_journal.yaml | 6 + qa/tasks/bucket_logging_tests.py | 24 +++- src/test/rgw/bucket_logging/README.rst | 13 ++ src/test/rgw/bucket_logging/conftest.py | 62 ++++++++- .../rgw/bucket_logging/test_bucket_logging.py | 120 +++++++++--------- 5 files changed, 164 insertions(+), 61 deletions(-) create mode 100644 qa/suites/rgw/bucket-logging/tasks/test_bucket_logging_journal.yaml diff --git a/qa/suites/rgw/bucket-logging/tasks/test_bucket_logging_journal.yaml b/qa/suites/rgw/bucket-logging/tasks/test_bucket_logging_journal.yaml new file mode 100644 index 000000000000..414ce4c000e7 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/tasks/test_bucket_logging_journal.yaml @@ -0,0 +1,6 @@ +tasks: +- tox: [client.0] +- bucket-logging-tests: + client.0: + rgw_server: client.0 + logging_type: Journal diff --git a/qa/tasks/bucket_logging_tests.py b/qa/tasks/bucket_logging_tests.py index c3203dd74473..55964278e4fa 100644 --- a/qa/tasks/bucket_logging_tests.py +++ b/qa/tasks/bucket_logging_tests.py @@ -41,12 +41,30 @@ def download(ctx, config): ], ) + # Journal mode requires the boto3 LoggingType extension + if client_config.get('logging_type') == 'Journal': + ctx.cluster.only(client).run( + args=['mkdir', '-p', '/home/ubuntu/.aws/models/s3/2006-03-01/'] + ) + (remote,) = ctx.cluster.only(client).remotes.keys() + remote_file = '/home/ubuntu/.aws/models/s3/2006-03-01/service-2.sdk-extras.json' + local_file = '{tdir}/ceph/examples/rgw/boto3/service-2.sdk-extras.json'.format(tdir=testdir) + remote.run(args=['cp', local_file, remote_file]) + try: yield finally: log.info('Removing bucket-logging-tests...') testdir = teuthology.get_testdir(ctx) - for client in config: + for client, client_config in config.items(): + if client_config.get('logging_type') == 'Journal': + ctx.cluster.only(client).run( + args=['rm', '-rf', '/home/ubuntu/.aws/models/s3/2006-03-01/service-2.sdk-extras.json'] + ) + ctx.cluster.only(client).run( + args=['cd', '/home/ubuntu/', run.Raw('&&'), + 'rmdir', '-p', '.aws/models/s3/2006-03-01/'] + ) ctx.cluster.only(client).run( args=[ 'rm', @@ -187,6 +205,10 @@ def run_tests(ctx, config): 'BUCKET_LOGGING_TESTS_CONF=./bucket-logging-tests.{client}.conf'.format(client=client), 'tox', '--', '-v', '-m', ' or '.join(attr)] + logging_type = client_config.get('logging_type') + if logging_type: + args.extend(['--logging-type', logging_type]) + toxvenv_sh(ctx, remote, args, label="bucket logging tests against rgw") yield diff --git a/src/test/rgw/bucket_logging/README.rst b/src/test/rgw/bucket_logging/README.rst index fc27135c18d3..00e03912e49f 100644 --- a/src/test/rgw/bucket_logging/README.rst +++ b/src/test/rgw/bucket_logging/README.rst @@ -33,6 +33,19 @@ From within the ``src/test/rgw/bucket_logging`` directory: BUCKET_LOGGING_TESTS_CONF=bucket_logging_tests.conf tox +By default, the tests run against ``Standard`` bucket logging mode. To run the +suite against ``Journal`` mode instead, pass the ``--logging-type`` flag: + +.. code-block:: bash + + BUCKET_LOGGING_TESTS_CONF=bucket_logging_tests.conf tox -- --logging-type=Journal + +Journal mode is a Ceph extension to the S3 bucket logging API. Running the +tests in Journal mode requires the boto3 ``LoggingType`` field, which is +provided by ``examples/rgw/boto3/service-2.sdk-extras.json``. Copy that file +to ``~/.aws/models/s3/2006-03-01/`` before running with ``--logging-type=Journal``; +otherwise the suite exits early with a clear error. + Test Coverage ============= diff --git a/src/test/rgw/bucket_logging/conftest.py b/src/test/rgw/bucket_logging/conftest.py index 89ca9cbbbfb7..164cddb12c63 100644 --- a/src/test/rgw/bucket_logging/conftest.py +++ b/src/test/rgw/bucket_logging/conftest.py @@ -1,6 +1,64 @@ import pytest -from . import setup +from botocore.exceptions import ClientError, ParamValidationError +from . import setup, get_config_host, get_config_port, get_access_key, get_secret_key +import boto3 + + +def _has_bucket_logging_extension(): + """Check if boto3 supports the LoggingType field (sdk-extras.json installed).""" + hostname = get_config_host() + port = get_config_port() + if port in (443, 8443): + endpoint_url = f'https://{hostname}:{port}' + else: + endpoint_url = f'http://{hostname}:{port}' + + client = boto3.client( + 's3', + endpoint_url=endpoint_url, + aws_access_key_id=get_access_key(), + aws_secret_access_key=get_secret_key(), + verify=False + ) + try: + client.put_bucket_logging( + Bucket='nonexistent-probe-bucket', + BucketLoggingStatus={ + 'LoggingEnabled': { + 'TargetBucket': 'nonexistent-probe-log', + 'TargetPrefix': 'log/', + 'LoggingType': 'Journal' + } + } + ) + except ParamValidationError: + return False + except ClientError: + return True + return True + + +def pytest_addoption(parser): + parser.addoption( + "--logging-type", + choices=["Standard", "Journal"], + default="Standard", + help="Bucket logging type to test: Standard (default) or Journal", + ) + @pytest.fixture(autouse=True, scope="session") -def setup_config(): +def setup_config(request): setup() + logging_type = request.config.getoption("--logging-type") + if logging_type == "Journal" and not _has_bucket_logging_extension(): + pytest.exit( + "Journal mode requested but boto3 LoggingType extension is not available. " + "Install service-2.sdk-extras.json to ~/.aws/models/s3/2006-03-01/", + returncode=1, + ) + + +@pytest.fixture(scope="session") +def logging_type(request): + return request.config.getoption("--logging-type") diff --git a/src/test/rgw/bucket_logging/test_bucket_logging.py b/src/test/rgw/bucket_logging/test_bucket_logging.py index 2244be9e74a7..f85aec6bc89c 100644 --- a/src/test/rgw/bucket_logging/test_bucket_logging.py +++ b/src/test/rgw/bucket_logging/test_bucket_logging.py @@ -192,25 +192,28 @@ def setup_logging_target(s3_client, log_bucket, source_bucket=None): return False -def enable_bucket_logging(s3_client, source_bucket, log_bucket, prefix=None): +def enable_bucket_logging(s3_client, source_bucket, log_bucket, prefix=None, logging_type='Standard'): """Enable logging on source bucket pointing to log bucket. Returns success status.""" if prefix is None: prefix = f'{source_bucket}/' try: + logging_enabled = { + 'TargetBucket': log_bucket, + 'TargetPrefix': prefix + } + if logging_type == 'Journal': + logging_enabled['LoggingType'] = 'Journal' s3_client.put_bucket_logging(Bucket=source_bucket, BucketLoggingStatus={ - 'LoggingEnabled': { - 'TargetBucket': log_bucket, - 'TargetPrefix': prefix - } + 'LoggingEnabled': logging_enabled }) - log.debug(f"Enabled logging on {source_bucket} -> {log_bucket} with prefix '{prefix}'") + log.debug(f"Enabled {logging_type} logging on {source_bucket} -> {log_bucket} with prefix '{prefix}'") return True except ClientError as e: log.error(f"Error enabling bucket logging: {e}") return False -def create_bucket_with_logging(s3_client, source_bucket, log_bucket): +def create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type='Standard'): """Create source/log buckets and enable logging. Returns success status.""" try: s3_client.create_bucket(Bucket=source_bucket) @@ -221,10 +224,10 @@ def create_bucket_with_logging(s3_client, source_bucket, log_bucket): if not setup_logging_target(s3_client, log_bucket, source_bucket): return False - return enable_bucket_logging(s3_client, source_bucket, log_bucket) + return enable_bucket_logging(s3_client, source_bucket, log_bucket, logging_type=logging_type) -def setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket): +def setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket, logging_type='Standard'): """Create two source buckets logging to the same log bucket. Returns success status.""" try: s3_client.create_bucket(Bucket=log_bucket) @@ -237,13 +240,16 @@ def setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_ s3_client.create_bucket(Bucket=source_bucket_2) for src in [source_bucket_1, source_bucket_2]: + logging_enabled = { + 'TargetBucket': log_bucket, + 'TargetPrefix': f'{src}/' + } + if logging_type == 'Journal': + logging_enabled['LoggingType'] = 'Journal' s3_client.put_bucket_logging( Bucket=src, BucketLoggingStatus={ - 'LoggingEnabled': { - 'TargetBucket': log_bucket, - 'TargetPrefix': f'{src}/' - } + 'LoggingEnabled': logging_enabled } ) @@ -272,7 +278,7 @@ def cleanup_bucket(s3_client, bucket_name): ##################### @pytest.mark.basic_test -def test_bucket_logging_list(s3_client): +def test_bucket_logging_list(s3_client, logging_type): """Test radosgw-admin bucket logging list command. Note: the 'list' command returns pending commit objects — log objects that have @@ -287,7 +293,7 @@ def test_bucket_logging_list(s3_client): log_bucket = gen_bucket_name("log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) upload_test_objects(s3_client, source_bucket) @@ -304,13 +310,13 @@ def test_bucket_logging_list(s3_client): @pytest.mark.basic_test -def test_bucket_logging_info_source(s3_client): +def test_bucket_logging_info_source(s3_client, logging_type): """Test radosgw-admin bucket logging info on source bucket.""" source_bucket = gen_bucket_name("source") log_bucket = gen_bucket_name("log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) @@ -327,13 +333,13 @@ def test_bucket_logging_info_source(s3_client): @pytest.mark.basic_test -def test_bucket_logging_info_log(s3_client): +def test_bucket_logging_info_log(s3_client, logging_type): """Test radosgw-admin bucket logging info on log bucket.""" source_bucket = gen_bucket_name("source") log_bucket = gen_bucket_name("log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', log_bucket]) @@ -348,13 +354,13 @@ def test_bucket_logging_info_log(s3_client): @pytest.mark.basic_test -def test_bucket_logging_flush(s3_client): +def test_bucket_logging_flush(s3_client, logging_type): """Test radosgw-admin bucket logging flush command.""" source_bucket = gen_bucket_name("source") log_bucket = gen_bucket_name("log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) upload_test_objects(s3_client, source_bucket) @@ -377,13 +383,13 @@ def test_bucket_logging_flush(s3_client): @pytest.mark.basic_test -def test_cleanup_on_log_bucket_delete(s3_client): +def test_cleanup_on_log_bucket_delete(s3_client, logging_type): """Test that temp log objects are deleted when log bucket is deleted.""" source_bucket = gen_bucket_name("cleanup-source") log_bucket = gen_bucket_name("cleanup-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) log_bucket_id = get_bucket_id(log_bucket) assert log_bucket_id is not None, f"Failed to get bucket ID for {log_bucket}" @@ -408,13 +414,13 @@ def test_cleanup_on_log_bucket_delete(s3_client): @pytest.mark.basic_test -def test_cleanup_on_logging_disable(s3_client): +def test_cleanup_on_logging_disable(s3_client, logging_type): """Test that disabling logging flushes pending logs to the log bucket.""" source_bucket = gen_bucket_name("disable-source") log_bucket = gen_bucket_name("disable-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) upload_test_objects(s3_client, source_bucket) @@ -438,22 +444,19 @@ def test_cleanup_on_logging_disable(s3_client): @pytest.mark.skip(reason="https://tracker.ceph.com/issues/75295") @pytest.mark.basic_test -def test_cleanup_on_logging_config_change(s3_client): +def test_cleanup_on_logging_config_change(s3_client, logging_type): """Test that changing logging target bucket implicitly flushes pending records to the old bucket.""" source_bucket = gen_bucket_name("config-change-source") log_bucket_1 = gen_bucket_name("config-change-log1") log_bucket_2 = gen_bucket_name("config-change-log2") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket_1) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket_1, logging_type) upload_test_objects(s3_client, source_bucket) assert setup_logging_target(s3_client, log_bucket_2, source_bucket) - - s3_client.put_bucket_logging(Bucket=source_bucket, BucketLoggingStatus={ - 'LoggingEnabled': {'TargetBucket': log_bucket_2, 'TargetPrefix': f'{source_bucket}/'} - }) + assert enable_bucket_logging(s3_client, source_bucket, log_bucket_2, logging_type=logging_type) time.sleep(2) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) @@ -474,13 +477,13 @@ def test_cleanup_on_logging_config_change(s3_client): @pytest.mark.basic_test -def test_cleanup_on_source_bucket_delete(s3_client): +def test_cleanup_on_source_bucket_delete(s3_client, logging_type): """Test that deleting source bucket flushes pending logs.""" source_bucket = gen_bucket_name("src-delete-source") log_bucket = gen_bucket_name("src-delete-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) upload_test_objects(s3_client, source_bucket) @@ -505,14 +508,14 @@ def test_cleanup_on_source_bucket_delete(s3_client): @pytest.mark.basic_test -def test_bucket_logging_info_log_multiple_sources(s3_client): +def test_bucket_logging_info_log_multiple_sources(s3_client, logging_type): """Test that multiple source buckets can log to the same log bucket.""" source_bucket_1 = gen_bucket_name("multi-source1") source_bucket_2 = gen_bucket_name("multi-source2") log_bucket = gen_bucket_name("multi-log") try: - assert setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket) + assert setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', log_bucket]) @@ -529,14 +532,14 @@ def test_bucket_logging_info_log_multiple_sources(s3_client): @pytest.mark.basic_test -def test_multiple_sources_disable_one(s3_client): +def test_multiple_sources_disable_one(s3_client, logging_type): """Test that disabling one source does not affect the other source's logging.""" source_bucket_1 = gen_bucket_name("disable-one-src1") source_bucket_2 = gen_bucket_name("disable-one-src2") log_bucket = gen_bucket_name("disable-one-log") try: - assert setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket) + assert setup_multi_source_logging(s3_client, source_bucket_1, source_bucket_2, log_bucket, logging_type) s3_client.put_bucket_logging(Bucket=source_bucket_1, BucketLoggingStatus={}) @@ -570,13 +573,13 @@ def test_multiple_sources_disable_one(s3_client): @pytest.mark.basic_test -def test_logging_info_after_disable(s3_client): +def test_logging_info_after_disable(s3_client, logging_type): """Verify that bucket logging info returns empty for source after logging is disabled.""" source_bucket = gen_bucket_name("info-disable-src") log_bucket = gen_bucket_name("info-disable-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) assert ret == 0 @@ -596,13 +599,13 @@ def test_logging_info_after_disable(s3_client): @pytest.mark.basic_test -def test_logging_info_after_source_delete(s3_client): +def test_logging_info_after_source_delete(s3_client, logging_type): """Verify that bucket logging info on deleted source returns error.""" source_bucket = gen_bucket_name("info-delete-src") log_bucket = gen_bucket_name("info-delete-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) assert ret == 0 @@ -622,13 +625,13 @@ def test_logging_info_after_source_delete(s3_client): @pytest.mark.basic_test -def test_flush_empty_creates_empty_object(s3_client): +def test_flush_empty_creates_empty_object(s3_client, logging_type): """Test that flushing with no pending data creates a size-zero committed log object.""" source_bucket = gen_bucket_name("empty-flush-src") log_bucket = gen_bucket_name("empty-flush-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'flush', '--bucket', source_bucket]) assert ret == 0, f"Flush failed with return code {ret}" @@ -648,7 +651,7 @@ def test_flush_empty_creates_empty_object(s3_client): @pytest.mark.basic_test -def test_logging_config_update_prefix(s3_client): +def test_logging_config_update_prefix(s3_client, logging_type): """Test that updating logging prefix is reflected in config and log objects.""" source_bucket = gen_bucket_name("update-prefix-src") log_bucket = gen_bucket_name("update-prefix-log") @@ -657,16 +660,14 @@ def test_logging_config_update_prefix(s3_client): new_prefix = "new-prefix/" try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) assert ret == 0 assert output.strip() assert parse_logging_config(output)['targetPrefix'] == old_prefix - s3_client.put_bucket_logging(Bucket=source_bucket, BucketLoggingStatus={ - 'LoggingEnabled': {'TargetBucket': log_bucket, 'TargetPrefix': new_prefix} - }) + assert enable_bucket_logging(s3_client, source_bucket, log_bucket, prefix=new_prefix, logging_type=logging_type) output, ret = admin(['bucket', 'logging', 'info', '--bucket', source_bucket]) assert ret == 0 @@ -679,26 +680,29 @@ def test_logging_config_update_prefix(s3_client): @pytest.mark.basic_test -def test_logging_config_change_key_format(s3_client): +def test_logging_config_change_key_format(s3_client, logging_type): """Test that changing obj_key_format implicitly flushes pending records to the same log bucket.""" source_bucket = gen_bucket_name("format-change-src") log_bucket = gen_bucket_name("format-change-log") try: - assert create_bucket_with_logging(s3_client, source_bucket, log_bucket) + assert create_bucket_with_logging(s3_client, source_bucket, log_bucket, logging_type) upload_test_objects(s3_client, source_bucket) - s3_client.put_bucket_logging(Bucket=source_bucket, BucketLoggingStatus={ - 'LoggingEnabled': { - 'TargetBucket': log_bucket, - 'TargetPrefix': f'{source_bucket}/', - 'TargetObjectKeyFormat': { - 'PartitionedPrefix': { - 'PartitionDateSource': 'DeliveryTime' - } + logging_enabled = { + 'TargetBucket': log_bucket, + 'TargetPrefix': f'{source_bucket}/', + 'TargetObjectKeyFormat': { + 'PartitionedPrefix': { + 'PartitionDateSource': 'DeliveryTime' } } + } + if logging_type == 'Journal': + logging_enabled['LoggingType'] = 'Journal' + s3_client.put_bucket_logging(Bucket=source_bucket, BucketLoggingStatus={ + 'LoggingEnabled': logging_enabled }) time.sleep(2) -- 2.47.3