From: sujay-d07 Date: Fri, 12 Jun 2026 08:23:59 +0000 (+0530) Subject: qa/rgw: Adding GSSAPI testing task for Bucket Notification Testing X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=93a357c93c08c9c7e489e2a87243a161307a0495;p=ceph.git qa/rgw: Adding GSSAPI testing task for Bucket Notification Testing This adds a Kerberos setup task, tells the Kafka task to emit GSSAPI listeners and broker principals, and updates notification tests to pass Kerberos credentials through to the bucket-notification suite. It enables the RGW Kafka notification coverage to run against GSSAPI-secured brokers. Signed-off-by: sujay-d07 --- diff --git a/qa/suites/rgw/notifications/overrides.yaml b/qa/suites/rgw/notifications/overrides.yaml index b5d6e94ba6c..733eb0c80e2 100644 --- a/qa/suites/rgw/notifications/overrides.yaml +++ b/qa/suites/rgw/notifications/overrides.yaml @@ -11,6 +11,7 @@ overrides: rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= rgw crypt require ssl: false rgw allow notification secrets in cleartext: true + rgw kafka sasl kerberos service name: kafka rgw: realm: MyRealm zonegroup: MyZoneGroup diff --git a/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml b/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml index 1acc2c74ca8..915b93199b2 100644 --- a/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml +++ b/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml @@ -1,4 +1,6 @@ tasks: +- kerberos: + client.0: - kafka: client.0: kafka_version: 3.9.2 diff --git a/qa/tasks/kafka.py b/qa/tasks/kafka.py index 7d3b4ec4bdc..0d2f095826d 100644 --- a/qa/tasks/kafka.py +++ b/qa/tasks/kafka.py @@ -42,6 +42,8 @@ def install_kafka(ctx, config): assert isinstance(config, dict) log.info('Installing Kafka...') + kerberos = getattr(ctx, 'kerberos', None) + # programmatically find a nearby mirror so as not to hammer archive.apache.org apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \ "grep -o '[^<]*' | sed 's/<[^>]*>//g' | head -n 1" @@ -101,7 +103,8 @@ def install_kafka(ctx, config): args=['sudo', 'chmod', 'o+rx', '/home/ubuntu'], ) - broker_conf(ctx, client, kafka_dir) + client_kerberos = kerberos.get(client) if kerberos else None + broker_conf(ctx, client, kafka_dir, client_kerberos) try: yield @@ -123,7 +126,7 @@ def install_kafka(ctx, config): ) -def broker_conf(ctx, client, kafka_dir): +def broker_conf(ctx, client, kafka_dir, kerberos): """writing a custom server.properties config file""" (remote,) = ctx.cluster.only(client).remotes.keys() ip = remote.ip_address @@ -170,8 +173,9 @@ def broker_conf(ctx, client, kafka_dir): "listener.name.mtls.ssl.truststore.location={tdir}/server.truststore.jks\n" "listener.name.mtls.ssl.truststore.password=mypassword\n" # SASL mechanisms - "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n" + "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI\n" "sasl.mechanism.inter.broker.protocol=PLAIN\n" + "sasl.kerberos.service.name={service_name}\n" 'listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n' ' username="admin" \\\n' ' password="admin-secret" \\\n' @@ -182,6 +186,11 @@ def broker_conf(ctx, client, kafka_dir): 'listener.name.sasl_ssl.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n' ' username="admin" \\\n' ' password="admin-secret";\n' + 'listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n' + ' useKeyTab=true \\\n' + ' storeKey=true \\\n' + ' keyTab="{keytab}" \\\n' + ' principal="{principal}";\n' 'listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n' ' username="admin" \\\n' ' password="admin-secret" \\\n' @@ -192,6 +201,11 @@ def broker_conf(ctx, client, kafka_dir): 'listener.name.sasl_plaintext.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n' ' username="admin" \\\n' ' password="admin-secret";\n' + 'listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n' + ' useKeyTab=true \\\n' + ' storeKey=true \\\n' + ' keyTab="{keytab}" \\\n' + ' principal="{principal}";\n' ).format( tdir=kafka_dir, ip=ip, @@ -200,6 +214,9 @@ def broker_conf(ctx, client, kafka_dir): sasl_ssl=KAFKA_PORTS['SASL_SSL'], sasl_plaintext=KAFKA_PORTS['SASL_PLAINTEXT'], mtls=KAFKA_PORTS['MTLS'], + service_name=kerberos['service_name'] if kerberos else '', + keytab=kerberos['kafka_keytab'] if kerberos else '', + principal=kerberos['kafka_principal'] if kerberos else '', ) file_name = 'server.properties' log.info("kafka conf file: %s", file_name) diff --git a/qa/tasks/kerberos.py b/qa/tasks/kerberos.py new file mode 100644 index 00000000000..5e68eaacea0 --- /dev/null +++ b/qa/tasks/kerberos.py @@ -0,0 +1,258 @@ +""" +Deploy and configure a Kerberos KDC for Teuthology. + +This task stands up a single-node MIT Kerberos KDC and creates the service +principals / keytabs used by the GSSAPI Kafka bucket-notification tests: + + - kafka/@CEPH.TEST (used by the Kafka broker) + - rgw/@CEPH.TEST (used by RGW and the test consumer) + +""" +import contextlib +import logging + +from teuthology import contextutil +from teuthology import misc as teuthology +from teuthology.orchestra import run + +log = logging.getLogger(__name__) + +REALM = 'CEPH.TEST' +MASTER_PASSWORD = 'ceph-kdc-master' +KEYTAB_DIR = '/etc/krb5-keytabs' +KAFKA_KEYTAB = '{}/kafka.service.keytab'.format(KEYTAB_DIR) +RGW_KEYTAB = '{}/rgw.service.keytab'.format(KEYTAB_DIR) +SERVICE_NAME = 'kafka' + + +def _distro_paths(remote): + """Return distro-specific KDC paths and service names.""" + if remote.os.package_type == 'rpm': + return { + 'kdc_conf_dir': '/var/kerberos/krb5kdc', + 'kdc_service': 'krb5kdc', + 'admin_service': 'kadmin', + } + return { + 'kdc_conf_dir': '/etc/krb5kdc', + 'kdc_service': 'krb5-kdc', + 'admin_service': 'krb5-admin-server', + } + + +@contextlib.contextmanager +def install_kerberos(ctx, config): + """Install the Kerberos KDC, admin server and GSSAPI SASL plugin.""" + assert isinstance(config, dict) + log.info('Installing Kerberos packages...') + + deb_packages = [ + 'krb5-kdc', 'krb5-admin-server', 'krb5-user', + 'libkrb5-dev', 'libsasl2-modules-gssapi-mit', + ] + rpm_packages = [ + 'krb5-devel', 'krb5-server', 'krb5-workstation', + 'cyrus-sasl-gssapi', + ] + + for (client, _) in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + if remote.os.package_type == 'rpm': + remote.run(args=['sudo', 'dnf', 'install', '-y'] + rpm_packages) + else: + remote.run(args=['sudo', 'apt-get', 'update']) + remote.run(args=[ + 'sudo', 'DEBIAN_FRONTEND=noninteractive', + 'apt-get', 'install', '-y', + ] + deb_packages) + + try: + yield + finally: + log.info('Removing Kerberos packages...') + for (client, _) in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + if remote.os.package_type == 'rpm': + remote.run( + args=['sudo', 'dnf', 'remove', '-y'] + rpm_packages, + check_status=False, + ) + else: + remote.run( + args=['sudo', 'DEBIAN_FRONTEND=noninteractive', + 'apt-get', 'purge', '-y'] + deb_packages, + check_status=False, + ) + + +@contextlib.contextmanager +def setup_kdc(ctx, config): + """Create the realm, principals and keytabs and start the KDC.""" + assert isinstance(config, dict) + log.info('Setting up Kerberos KDC...') + + if not hasattr(ctx, 'kerberos'): + ctx.kerberos = {} + + for (client, _) in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + try: + ip = remote.ip_address + paths = _distro_paths(remote) + kafka_principal = 'kafka/{ip}@{realm}'.format(ip=ip, realm=REALM) + rgw_principal = 'rgw/{ip}@{realm}'.format(ip=ip, realm=REALM) + + # /etc/krb5.conf: rdns=false keeps the SPN IP-based (no reverse + # lookup to a hostname); dns_lookup_kdc/realm=false keeps the KDC + # and realm settings local and static, avoiding any DNS-related + # issues in the test environment. + krb5_conf = ( + "[libdefaults]\n" + " default_realm = {realm}\n" + " dns_lookup_kdc = false\n" + " dns_lookup_realm = false\n" + " rdns = false\n" + " udp_preference_limit = 1\n" + "\n" + "[realms]\n" + " {realm} = {{\n" + " kdc = {ip}\n" + " admin_server = {ip}\n" + " }}\n" + ).format(realm=REALM, ip=ip) + remote.sudo_write_file('/etc/krb5.conf', krb5_conf) + + # kdc.conf + kdc_conf = ( + "[kdcdefaults]\n" + " kdc_ports = 88\n" + " kdc_tcp_ports = 88\n" + "\n" + "[realms]\n" + " {realm} = {{\n" + " max_life = 24h 0m 0s\n" + " max_renewable_life = 7d 0h 0m 0s\n" + " }}\n" + ).format(realm=REALM) + remote.run(args=['sudo', 'mkdir', '-p', paths['kdc_conf_dir']]) + remote.sudo_write_file( + '{}/kdc.conf'.format(paths['kdc_conf_dir']), kdc_conf) + remote.sudo_write_file( + '{}/kadm5.acl'.format(paths['kdc_conf_dir']), + '*/admin@{realm} *\n'.format(realm=REALM)) + + # create the KDC database + remote.run(args=[ + 'sudo', 'kdb5_util', 'create', '-s', '-r', REALM, + '-P', MASTER_PASSWORD, + ]) + + # create principals and extract keytabs + remote.run(args=[ + 'sudo', 'kadmin.local', '-q', + 'addprinc -randkey {}'.format(kafka_principal), + ]) + remote.run(args=[ + 'sudo', 'kadmin.local', '-q', + 'addprinc -randkey {}'.format(rgw_principal), + ]) + remote.run(args=['sudo', 'mkdir', '-p', KEYTAB_DIR]) + remote.run(args=[ + 'sudo', 'kadmin.local', '-q', + 'ktadd -k {keytab} {principal}'.format( + keytab=KAFKA_KEYTAB, principal=kafka_principal), + ]) + remote.run(args=[ + 'sudo', 'kadmin.local', '-q', + 'ktadd -k {keytab} {principal}'.format( + keytab=RGW_KEYTAB, principal=rgw_principal), + ]) + # make the keytabs world-readable to avoid "keytab contains + # no suitable keys" permission failures. + remote.run(args=[ + 'sudo', 'chmod', '644', KAFKA_KEYTAB, RGW_KEYTAB, + ]) + + # restart the KDC and admin server + remote.run(args=[ + 'sudo', 'systemctl', 'restart', + paths['kdc_service'], paths['admin_service'], + ]) + + # kinit with the RGW principal and list the tickets + remote.run(args=[ + 'kinit', '-kt', RGW_KEYTAB, rgw_principal, run.Raw('&&'), 'klist', + ]) + + ctx.kerberos[client] = { + 'realm': REALM, + 'service_name': SERVICE_NAME, + 'principal': rgw_principal, + 'keytab': RGW_KEYTAB, + 'kafka_principal': kafka_principal, + 'kafka_keytab': KAFKA_KEYTAB, + 'ip': ip, + } + except Exception as exc: + log.exception('Kerberos setup failed for %s', client) + raise RuntimeError('Kerberos setup is failing') from exc + + try: + yield + finally: + log.info('Tearing down Kerberos KDC...') + for (client, _) in config.items(): + (remote,) = ctx.cluster.only(client).remotes.keys() + paths = _distro_paths(remote) + remote.run( + args=['sudo', 'systemctl', 'stop', + paths['kdc_service'], paths['admin_service']], + check_status=False, + ) + remote.run( + args=['sudo', 'kdb5_util', 'destroy', '-f'], + check_status=False, + ) + remote.run( + args=['sudo', 'rm', '-rf', KEYTAB_DIR, '/etc/krb5.conf', + '/tmp/krb5cc_bntests'], + check_status=False, + ) + ctx.kerberos.pop(client, None) + + +@contextlib.contextmanager +def task(ctx, config): + """ + Set up a Kerberos KDC for the GSSAPI Kafka bucket-notification tests. + + Must run before the kafka and notification-tests tasks:: + + tasks: + - kerberos: + client.0: + - kafka: + client.0: + kafka_version: 3.9.2 + - notification-tests: + client.0: + extra_attr: ["kafka_test", "kafka_security_test"] + """ + assert config is None or isinstance(config, list) \ + or isinstance(config, dict), \ + "task kerberos only supports a list or dictionary for configuration" + + all_clients = ['client.{id}'.format(id=id_) + for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] + if config is None: + config = all_clients + if isinstance(config, list): + config = dict.fromkeys(config) + + log.debug('Kerberos config is %s', config) + + with contextutil.nested( + lambda: install_kerberos(ctx=ctx, config=config), + lambda: setup_kdc(ctx=ctx, config=config), + ): + yield diff --git a/qa/tasks/notification_tests.py b/qa/tasks/notification_tests.py index cd1b4752c3b..5f258e219c3 100644 --- a/qa/tasks/notification_tests.py +++ b/qa/tasks/notification_tests.py @@ -233,6 +233,15 @@ def run_tests(ctx, config): if kafka_dir: args.append('KAFKA_DIR={kafka_dir}'.format(kafka_dir=kafka_dir)) + # when a KDC has been set up, point the GSSAPI client (the kafka-python + # consumer used by the tests) at the rgw keytab so it can auto-acquire + # and auto-renew its kerberos credentials without a separate kinit. + kerberos = getattr(ctx, 'kerberos', None) + if kerberos and client in kerberos: + keytab = kerberos[client]['keytab'] + args.append('KRB5_CLIENT_KTNAME={keytab}'.format(keytab=keytab)) + args.append('KRB5CCNAME=FILE:/tmp/krb5cc_bntests') + args.extend([ '{tdir}/ceph/src/test/rgw/bucket_notification/virtualenv/bin/python'.format(tdir=testdir), '-m', 'pytest', @@ -251,10 +260,13 @@ def run_tests(ctx, config): @contextlib.contextmanager def task(ctx,config): """ - To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server. Also you need to pass the - 'extra_attr' to the notification tests. Following is the way how to run kafka and finally bucket notification tests:: + To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server and have kerberos setup. + Also you need to pass the 'extra_attr' to the notification tests. Following is the way how to run kerberos, kafka, and + finally bucket notification tests:: tasks: + - kerberos: + client.0: - kafka: client.0: kafka_version: 2.6.0 @@ -294,13 +306,25 @@ def task(ctx,config): bntests_conf = {} + kerberos = getattr(ctx, 'kerberos', None) + for client in clients: endpoint = ctx.rgw.role_endpoints.get(client) assert endpoint, 'bntests: no rgw endpoint for {}'.format(client) + if kerberos and client in kerberos: + kerberos_conf = { + 'service_name': kerberos[client]['service_name'], + 'principal': kerberos[client]['principal'], + 'keytab': kerberos[client]['keytab'], + } + else: + kerberos_conf = {'service_name': '', 'principal': '', 'keytab': ''} + cluster_name, _, _ = teuthology.split_role(client) bntests_conf[client] = ConfigObj( indent_type='', + write_empty_values=True, infile={ 'DEFAULT': { @@ -310,7 +334,8 @@ def task(ctx,config): 'cluster':cluster_name, 'version':'v2' }, - 's3 main':{} + 's3 main':{}, + 'kerberos':kerberos_conf } ) diff --git a/src/test/rgw/bucket_notification/bootstrap b/src/test/rgw/bucket_notification/bootstrap index 4d680c86d2f..535935426b0 100755 --- a/src/test/rgw/bucket_notification/bootstrap +++ b/src/test/rgw/bucket_notification/bootstrap @@ -2,7 +2,7 @@ set -e if [ -f /etc/debian_version ]; then - for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev; do + for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev libkrb5-dev; do if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then # add a space after old values missing="${missing:+$missing }$package" @@ -14,7 +14,7 @@ if [ -f /etc/debian_version ]; then fi fi if [ -f /etc/redhat-release ]; then - for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel; do + for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel krb5-devel; do if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then missing="${missing:+$missing }$package" fi