From: sujay-d07 Date: Mon, 1 Jun 2026 01:25:57 +0000 (+0530) Subject: rgw/testing: add GSSAPI testing in bucket notification tests X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=803f698200edd784e3eaab86cb18aeee05b1a5ed;p=ceph.git rgw/testing: add GSSAPI testing in bucket notification tests Signed-off-by: sujay-d07 --- diff --git a/src/test/rgw/bucket_notification/README.rst b/src/test/rgw/bucket_notification/README.rst index 6460e692507..59b3f5818e1 100644 --- a/src/test/rgw/bucket_notification/README.rst +++ b/src/test/rgw/bucket_notification/README.rst @@ -108,7 +108,8 @@ Kafka Security Tests listener.name.mtls.ssl.truststore.password=mypassword # SASL mechanisms - sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512 + sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI + sasl.kerberos.service.name=kafka sasl.mechanism.inter.broker.protocol=PLAIN inter.broker.listener.name=PLAINTEXT @@ -128,6 +129,13 @@ Kafka Security Tests username="admin" \ password="admin-secret"; + # SASL over SSL with GSSAPI mechanism + listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \ + useKeyTab=true \ + storeKey=true \ + keyTab="/etc/krb5-keytabs/kafka.service.keytab" \ + principal="kafka/192.168.1.100@REALM"; + # PLAINTEXT SASL with PLAIN mechanism listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \ username="admin" \ @@ -144,18 +152,109 @@ Kafka Security Tests username="admin" \ password="admin-secret"; -3. Start Zookeeper and Kafka. + # PLAINTEXT SASL with GSSAPI mechanism + listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \ + useKeyTab=true \ + storeKey=true \ + keyTab="/etc/krb5-keytabs/kafka.service.keytab" \ + principal="kafka/192.168.1.100@REALM"; + +3. Kerberos setup:: + + librdkafka (inside RGW) builds the broker SPN (Service Principal Name, basically the + principal that represents the Kafka broker) as ``kafka/@REALM``, + where ```` is whatever appears in ``advertised.listeners``. + Since step 2 uses the broker's IP in ``advertised.listeners``, the + broker principal must be tied to that same IP (or hostname — they must match). + + The examples below use realm ``REALM.COM`` and broker IP ``192.168.1.100``. + Substitute your own values. + + a. Install Kerberos and SASL/GSSAPI packages. + + Ubuntu/Debian:: + + sudo apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y krb5-kdc krb5-admin-server krb5-user \ + libsasl2-modules-gssapi-mit libsasl2-modules + + Fedora/RHEL:: + + sudo dnf install -y krb5-server krb5-workstation cyrus-sasl-gssapi + + b. Configure ``/etc/krb5.conf``. + + Reference contents:: + + [libdefaults] + default_realm = REALM.COM + dns_lookup_realm = false + dns_lookup_kdc = false + rdns = false + forwardable = true + ticket_lifetime = 24h + renew_lifetime = 7d + [realms] + REALM.COM = { + kdc = 192.168.1.100 + admin_server = 192.168.1.100 + } + + [domain_realm] + .realm.com = REALM.COM + realm.com = REALM.COM -4. Start RGW vstart cluster with cleartext parameter set to true:: + c. Bootstrap the KDC (first-time setup only):: + + sudo kdb5_util create -s -r REALM.COM # set + remember master password + echo '*/admin@REALM.COM *' | sudo tee /etc/krb5kdc/kadm5.acl # allow admin principal to do anything + + # start and enable KDC and admin server + Ubuntu/Debian:: + sudo systemctl enable --now krb5-kdc krb5-admin-server + sudo systemctl status krb5-kdc krb5-admin-server + + Fedora/RHEL:: + sudo systemctl enable --now krb5kdc kadmin + sudo systemctl status krb5kdc kadmin + + d. Create the broker and RGW principals + keytabs:: + + sudo mkdir -p /etc/krb5-keytabs # to store the keytabs + sudo kadmin.local -q "addprinc admin/admin" + sudo kadmin.local -q "addprinc -randkey kafka/192.168.1.100" + sudo kadmin.local -q "addprinc -randkey rgw/192.168.1.100" + sudo kadmin.local -q "ktadd -k /etc/krb5-keytabs/kafka.service.keytab kafka/192.168.1.100" + sudo kadmin.local -q "ktadd -k /etc/krb5-keytabs/rgw.service.keytab rgw/192.168.1.100" + + sudo chmod 640 /etc/krb5-keytabs/*.keytab + sudo chown root:$USER /etc/krb5-keytabs/*.keytab # Make keytabs readable by the user that runs the Kafka broker + # AND the user that runs vstart (often the same user) + + e. Verify the keytabs and get a TGT for the RGW principal:: + + klist -kt /etc/krb5-keytabs/kafka.service.keytab + klist -kt /etc/krb5-keytabs/rgw.service.keytab + kinit -kt /etc/krb5-keytabs/rgw.service.keytab rgw/192.168.1.100@REALM.COM + klist # should show a valid TGT + + f. Make sure to add the Kafka principal, RGW principal, and keytab to your ``bntests.conf``. + +4. Start Zookeeper and Kafka. + +5. Start RGW vstart cluster with cleartext parameter set to true:: cd /path/to/ceph/build - MON=1 OSD=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -d -o "rgw_allow_notification_secrets_in_cleartext=true" + MON=1 OSD=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -d -o "rgw_allow_notification_secrets_in_cleartext=true" -o "rgw_kafka_sasl_kerberos_service_name=kafka" -5. Run the tests:: +6. Run the tests:: cd /path/to/ceph KAFKA_DIR=/path/to/kafka BNTESTS_CONF=/path/to/bntests.conf python -m pytest -s /path/to/ceph/src/test/rgw/bucket_notification/test_bn.py -v -m 'kafka_security_test' + For GSSAPI tests, ensure a valid ticket cache already exists for RGW principal + (for example, via ``kinit``). + ============== RabbitMQ Tests ============== diff --git a/src/test/rgw/bucket_notification/__init__.py b/src/test/rgw/bucket_notification/__init__.py index 954c6575bb6..e15545ff9f9 100644 --- a/src/test/rgw/bucket_notification/__init__.py +++ b/src/test/rgw/bucket_notification/__init__.py @@ -18,6 +18,8 @@ def setup(): raise RuntimeError('Your config file is missing the DEFAULT section!') if not cfg.has_section("s3 main"): raise RuntimeError('Your config file is missing the "s3 main" section!') + if not cfg.has_section("kerberos"): + raise RuntimeError('Your config file is missing the "kerberos" section!') defaults = cfg.defaults() @@ -50,6 +52,15 @@ def setup(): global main_secret_key main_secret_key = cfg.get('s3 main',"secret_key") + global kerberos_service_name + kerberos_service_name = cfg.get('kerberos', 'service_name') + + global kerberos_principal + kerberos_principal = cfg.get('kerberos', 'principal') + + global kerberos_keytab + kerberos_keytab = cfg.get('kerberos', 'keytab') + def get_config_host(): global default_host return default_host @@ -74,6 +85,12 @@ def get_secret_key(): global main_secret_key return main_secret_key +def get_kerberos_config(): + global kerberos_service_name + global kerberos_principal + global kerberos_keytab + return kerberos_service_name, kerberos_principal, kerberos_keytab + @pytest.fixture(autouse=True, scope="package") def configfile(): setup() diff --git a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE index 2ab614c5a3e..85f643b6fd5 100644 --- a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE +++ b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE @@ -11,3 +11,8 @@ secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== display_name = M. Tester user_id = testid email = tester@ceph.com + +[kerberos] +service_name = kafka +principal = rgw/@REALM.COM +keytab = /etc/krb5-keytabs/rgw.service.keytab diff --git a/src/test/rgw/bucket_notification/bntests.conf.multisite b/src/test/rgw/bucket_notification/bntests.conf.multisite index f09e5389b20..cce59d8f5a7 100644 --- a/src/test/rgw/bucket_notification/bntests.conf.multisite +++ b/src/test/rgw/bucket_notification/bntests.conf.multisite @@ -11,3 +11,8 @@ secret_key = crayon display_name = RegularUser user_id = regular.user email = + +[kerberos] +service_name = kafka +principal = rgw/@REALM.COM +keytab = /etc/krb5-keytabs/rgw.service.keytab \ No newline at end of file diff --git a/src/test/rgw/bucket_notification/kafka-security.sh b/src/test/rgw/bucket_notification/kafka-security.sh index b3d216b0cd2..07e0dd05980 100755 --- a/src/test/rgw/bucket_notification/kafka-security.sh +++ b/src/test/rgw/bucket_notification/kafka-security.sh @@ -8,6 +8,9 @@ REQFILE=$FQDN.req CERTFILE=$FQDN.crt MYPW=mypassword VALIDITY=36500 +CLIENT_KEYFILE=client.key +CLIENT_CERTFILE=client.crt +CLIENT_REQFILE=client.req rm -f $KEYFILE rm -f $TRUSTFILE @@ -66,9 +69,6 @@ keytool -storepass $MYPW -keystore $KEYFILE -alias localhost \ -import -file $CERTFILE echo "########## generate client certificate for mTLS testing" -CLIENT_KEYFILE=client.key -CLIENT_CERTFILE=client.crt -CLIENT_REQFILE=client.req # generate client private key (PKCS#8 for compatibility) openssl genpkey -algorithm RSA -out $CLIENT_KEYFILE -pkeyopt rsa_keygen_bits:2048 diff --git a/src/test/rgw/bucket_notification/requirements.txt b/src/test/rgw/bucket_notification/requirements.txt index 55685e4842d..8c1d0ad8807 100644 --- a/src/test/rgw/bucket_notification/requirements.txt +++ b/src/test/rgw/bucket_notification/requirements.txt @@ -6,3 +6,4 @@ pika cloudevents>=1.12.1,<2 xmltodict requests>=2.28.0 +gssapi>=1.8.0 diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py index fb941e7185b..0f22b4ffb4d 100644 --- a/src/test/rgw/bucket_notification/test_bn.py +++ b/src/test/rgw/bucket_notification/test_bn.py @@ -30,7 +30,8 @@ from . import( get_config_zonegroup, get_config_cluster, get_access_key, - get_secret_key + get_secret_key, + get_kerberos_config, ) from .api import PSTopicS3, \ @@ -426,6 +427,12 @@ default_kafka_server = get_ip() KAFKA_TEST_USER = 'alice' KAFKA_TEST_PASSWORD = 'alice-secret' + +def get_kerberos_env(): + service_name, principal, keytab = get_kerberos_config() + return service_name, principal, keytab + + def setup_scram_users_via_kafka_configs(mechanism: str) -> None: """to setup SCRAM users using kafka-configs.sh after Kafka is running.""" if not mechanism.startswith('SCRAM'): @@ -552,17 +559,27 @@ class KafkaReceiver(object): if ca_cert: base_config['ssl_cafile'] = ca_cert base_config['sasl_mechanism'] = mechanism - base_config.update({ - 'sasl_plain_username': KAFKA_TEST_USER, - 'sasl_plain_password': KAFKA_TEST_PASSWORD, - }) + if mechanism == 'GSSAPI': + kerberos_service_name, _, _ = get_kerberos_env() + if kerberos_service_name: + base_config['sasl_kerberos_service_name'] = kerberos_service_name + else: + base_config.update({ + 'sasl_plain_username': KAFKA_TEST_USER, + 'sasl_plain_password': KAFKA_TEST_PASSWORD, + }) elif effective_protocol == 'SASL_PLAINTEXT': base_config['security_protocol'] = 'SASL_PLAINTEXT' base_config['sasl_mechanism'] = mechanism - base_config.update({ - 'sasl_plain_username': KAFKA_TEST_USER, - 'sasl_plain_password': KAFKA_TEST_PASSWORD, - }) + if mechanism == 'GSSAPI': + kerberos_service_name, _, _ = get_kerberos_env() + if kerberos_service_name: + base_config['sasl_kerberos_service_name'] = kerberos_service_name + else: + base_config.update({ + 'sasl_plain_username': KAFKA_TEST_USER, + 'sasl_plain_password': KAFKA_TEST_PASSWORD, + }) remaining_retries = 10 while remaining_retries > 0: @@ -4732,6 +4749,19 @@ def kafka_security(security_type, mechanism='PLAIN', use_topic_attrs_for_creds=F if mechanism.startswith('SCRAM'): setup_scram_users_via_kafka_configs(mechanism) time.sleep(2) # Allow time for SCRAM config to propagate + elif mechanism == 'GSSAPI': + service_name, principal, keytab = get_kerberos_env() + missing = [] + if not service_name: + missing.append('service_name') + if not principal: + missing.append('principal') + if not keytab: + missing.append('keytab') + if missing: + pytest.skip('Missing GSSAPI options in [kerberos] section of BNTESTS_CONF: ' + ', '.join(missing)) + if not os.path.isfile(keytab): + pytest.skip(f'[kerberos] keytab does not exist: {keytab}') conn = connection() zonegroup = get_config_zonegroup() @@ -4742,26 +4772,45 @@ def kafka_security(security_type, mechanism='PLAIN', use_topic_attrs_for_creds=F topic_name = bucket_name+'_topic' # create topic if security_type == 'SASL_SSL': - if not use_topic_attrs_for_creds: - endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9094' - else: + if mechanism == 'GSSAPI' or use_topic_attrs_for_creds: endpoint_address = 'kafka://' + default_kafka_server + ':9094' + else: + endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9094' elif security_type == 'SSL': if use_mtls: endpoint_address = 'kafka://' + default_kafka_server + ':9096' else: endpoint_address = 'kafka://' + default_kafka_server + ':9093' elif security_type == 'SASL_PLAINTEXT': - endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9095' + if mechanism == 'GSSAPI': + endpoint_address = 'kafka://' + default_kafka_server + ':9095' + else: + endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9095' else: assert False, 'unknown security method '+security_type if security_type == 'SASL_PLAINTEXT': endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker&use-ssl=false&mechanism='+mechanism + if mechanism == 'GSSAPI': + kerberos_service_name, kerberos_principal, kerberos_keytab = get_kerberos_env() + if kerberos_service_name: + endpoint_args += '&sasl-kerberos-service-name=' + kerberos_service_name + if kerberos_principal: + endpoint_args += '&sasl-kerberos-principal=' + kerberos_principal + if kerberos_keytab: + endpoint_args += '&sasl-kerberos-keytab=' + kerberos_keytab elif security_type == 'SASL_SSL': kafka_cert_dir = _kafka_cert_dir() endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker&use-ssl=true&ca-location='+kafka_cert_dir+'/y-ca.crt&mechanism='+mechanism - if use_topic_attrs_for_creds: + if mechanism == 'GSSAPI': + kerberos_service_name, kerberos_principal, kerberos_keytab = get_kerberos_env() + if kerberos_service_name: + endpoint_args += '&sasl-kerberos-service-name=' + kerberos_service_name + if kerberos_principal: + endpoint_args += '&sasl-kerberos-principal=' + kerberos_principal + if kerberos_keytab: + endpoint_args += '&sasl-kerberos-keytab=' + kerberos_keytab + elif use_topic_attrs_for_creds: endpoint_args += '&user-name=alice&password=alice-secret' else: kafka_cert_dir = _kafka_cert_dir() @@ -4886,6 +4935,16 @@ def test_notification_kafka_security_ssl_sasl_scram_512(): kafka_security('SASL_SSL', mechanism='SCRAM-SHA-512') +@pytest.mark.kafka_security_test +def test_notification_kafka_security_sasl_gssapi(): + kafka_security('SASL_PLAINTEXT', mechanism='GSSAPI') + + +@pytest.mark.kafka_security_test +def test_notification_kafka_security_ssl_sasl_gssapi(): + kafka_security('SASL_SSL', mechanism='GSSAPI') + + @pytest.mark.kafka_security_test def test_notification_kafka_security_ssl_mtls(): """test mTLS client certificate authentication to Kafka"""