rgw/testing: add GSSAPI testing in bucket notification tests

Signed-off-by: sujay-d07 <sujaydongre07@gmail.com>

diff --git a/src/test/rgw/bucket_notification/README.rst b/src/test/rgw/bucket_notification/README.rst
index 6460e692507e1fb5de9ffc16e39aaac117130d2b..59b3f5818e1fba97f4fdcf3c2d973f9e0f05c635 100644 (file)
--- a/src/test/rgw/bucket_notification/README.rst
+++ b/src/test/rgw/bucket_notification/README.rst
@@ -108,7 +108,8 @@ Kafka Security Tests
         listener.name.mtls.ssl.truststore.password=mypassword
 
         # SASL mechanisms
-        sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
+        sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
+        sasl.kerberos.service.name=kafka
         sasl.mechanism.inter.broker.protocol=PLAIN
         inter.broker.listener.name=PLAINTEXT
 
@@ -128,6 +129,13 @@ Kafka Security Tests
         username="admin" \
         password="admin-secret";
 
+        # SASL over SSL with GSSAPI mechanism
+        listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \
+        useKeyTab=true \
+        storeKey=true \
+        keyTab="/etc/krb5-keytabs/kafka.service.keytab" \
+        principal="kafka/192.168.1.100@REALM";
+
         # PLAINTEXT SASL with PLAIN mechanism
         listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
         username="admin" \
@@ -144,18 +152,109 @@ Kafka Security Tests
         username="admin" \
         password="admin-secret";
 
-3. Start Zookeeper and Kafka.
+        # PLAINTEXT SASL with GSSAPI mechanism
+        listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \
+        useKeyTab=true \
+        storeKey=true \
+        keyTab="/etc/krb5-keytabs/kafka.service.keytab" \
+        principal="kafka/192.168.1.100@REALM";
+
+3. Kerberos setup::
+
+   librdkafka (inside RGW) builds the broker SPN (Service Principal Name, basically the 
+   principal that represents the Kafka broker) as ``kafka/<broker-host>@REALM``, 
+   where ``<broker-host>`` is whatever appears in ``advertised.listeners``.
+   Since step 2 uses the broker's IP in ``advertised.listeners``, the 
+   broker principal must be tied to that same IP (or hostname — they must match).
+
+   The examples below use realm ``REALM.COM`` and broker IP ``192.168.1.100``.
+   Substitute your own values.
+
+   a. Install Kerberos and SASL/GSSAPI packages.
+
+      Ubuntu/Debian::
+
+              sudo apt update
+              sudo DEBIAN_FRONTEND=noninteractive apt install -y krb5-kdc krb5-admin-server krb5-user \
+                                  libsasl2-modules-gssapi-mit libsasl2-modules
+
+      Fedora/RHEL::
+
+              sudo dnf install -y krb5-server krb5-workstation cyrus-sasl-gssapi
+
+   b. Configure ``/etc/krb5.conf``.
+
+      Reference contents::
+
+              [libdefaults]
+                        default_realm = REALM.COM
+                        dns_lookup_realm = false
+                        dns_lookup_kdc = false
+                        rdns = false
+                        forwardable = true
+                        ticket_lifetime = 24h
+                        renew_lifetime = 7d
+              [realms]
+                        REALM.COM = {
+                                kdc = 192.168.1.100
+                                admin_server = 192.168.1.100
+                        }
+
+              [domain_realm]
+                        .realm.com = REALM.COM
+                        realm.com = REALM.COM
 
-4. Start RGW vstart cluster with cleartext parameter set to true::
+   c. Bootstrap the KDC (first-time setup only)::
+
+              sudo kdb5_util create -s -r REALM.COM                                # set + remember master password
+              echo '*/admin@REALM.COM    *' | sudo tee /etc/krb5kdc/kadm5.acl      # allow admin principal to do anything
+
+              # start and enable KDC and admin server
+              Ubuntu/Debian::
+                      sudo systemctl enable --now krb5-kdc krb5-admin-server
+                      sudo systemctl status krb5-kdc krb5-admin-server
+
+              Fedora/RHEL::
+                      sudo systemctl enable --now krb5kdc kadmin
+                      sudo systemctl status krb5kdc kadmin
+
+   d. Create the broker and RGW principals + keytabs::
+
+              sudo mkdir -p /etc/krb5-keytabs                                      # to store the keytabs
+              sudo kadmin.local -q "addprinc admin/admin"
+              sudo kadmin.local -q "addprinc -randkey kafka/192.168.1.100"
+              sudo kadmin.local -q "addprinc -randkey rgw/192.168.1.100"
+              sudo kadmin.local -q "ktadd -k /etc/krb5-keytabs/kafka.service.keytab kafka/192.168.1.100"
+              sudo kadmin.local -q "ktadd -k /etc/krb5-keytabs/rgw.service.keytab rgw/192.168.1.100"
+
+              sudo chmod 640 /etc/krb5-keytabs/*.keytab
+              sudo chown root:$USER /etc/krb5-keytabs/*.keytab                     # Make keytabs readable by the user that runs the Kafka broker 
+                                                                                   # AND the user that runs vstart (often the same user)
+
+   e. Verify the keytabs and get a TGT for the RGW principal::
+
+              klist -kt /etc/krb5-keytabs/kafka.service.keytab
+              klist -kt /etc/krb5-keytabs/rgw.service.keytab
+              kinit -kt /etc/krb5-keytabs/rgw.service.keytab rgw/192.168.1.100@REALM.COM
+              klist                                                                # should show a valid TGT
+
+   f. Make sure to add the Kafka principal, RGW principal, and keytab to your ``bntests.conf``.
+
+4. Start Zookeeper and Kafka.
+
+5. Start RGW vstart cluster with cleartext parameter set to true::
 
         cd /path/to/ceph/build
-        MON=1 OSD=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -d -o "rgw_allow_notification_secrets_in_cleartext=true"
+        MON=1 OSD=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -d -o "rgw_allow_notification_secrets_in_cleartext=true" -o "rgw_kafka_sasl_kerberos_service_name=kafka"
 
-5. Run the tests::
+6. Run the tests::
 
         cd /path/to/ceph
         KAFKA_DIR=/path/to/kafka BNTESTS_CONF=/path/to/bntests.conf python -m pytest -s /path/to/ceph/src/test/rgw/bucket_notification/test_bn.py -v -m 'kafka_security_test'
 
+        For GSSAPI tests, ensure a valid ticket cache already exists for RGW principal
+        (for example, via ``kinit``).
+
 ==============
 RabbitMQ Tests
 ==============

diff --git a/src/test/rgw/bucket_notification/__init__.py b/src/test/rgw/bucket_notification/__init__.py
index 954c6575bb6c3d7ea0535ec41654e542ede92035..e15545ff9f9bdd14ec83f0049b5cfd658dfd33bc 100644 (file)
--- a/src/test/rgw/bucket_notification/__init__.py
+++ b/src/test/rgw/bucket_notification/__init__.py
@@ -18,6 +18,8 @@ def setup():
         raise RuntimeError('Your config file is missing the DEFAULT section!')
     if not cfg.has_section("s3 main"):
         raise RuntimeError('Your config file is missing the "s3 main" section!')
+    if not cfg.has_section("kerberos"):
+        raise RuntimeError('Your config file is missing the "kerberos" section!')
 
     defaults = cfg.defaults()
 
@@ -50,6 +52,15 @@ def setup():
     global main_secret_key
     main_secret_key = cfg.get('s3 main',"secret_key")
 
+    global kerberos_service_name
+    kerberos_service_name = cfg.get('kerberos', 'service_name')
+
+    global kerberos_principal
+    kerberos_principal = cfg.get('kerberos', 'principal')
+
+    global kerberos_keytab
+    kerberos_keytab = cfg.get('kerberos', 'keytab')
+
 def get_config_host():
     global default_host
     return default_host
@@ -74,6 +85,12 @@ def get_secret_key():
     global main_secret_key
     return main_secret_key
 
+def get_kerberos_config():
+    global kerberos_service_name
+    global kerberos_principal
+    global kerberos_keytab
+    return kerberos_service_name, kerberos_principal, kerberos_keytab
+
 @pytest.fixture(autouse=True, scope="package")
 def configfile():
     setup()

diff --git a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
index 2ab614c5a3e721606ad0f735567aa7a9db735c8c..85f643b6fd52f83ac8ba0fcf47451e6b73f0c737 100644 (file)
--- a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
+++ b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
@@ -11,3 +11,8 @@ secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
 display_name = M. Tester
 user_id = testid
 email = tester@ceph.com
+
+[kerberos]
+service_name = kafka
+principal = rgw/<broker-fqdn/ip>@REALM.COM
+keytab = /etc/krb5-keytabs/rgw.service.keytab

diff --git a/src/test/rgw/bucket_notification/bntests.conf.multisite b/src/test/rgw/bucket_notification/bntests.conf.multisite
index f09e5389b20f8a35e197a9d41aaf5a75f5159b55..cce59d8f5a7b67df4c9b9d50cf95ecf4debefaa1 100644 (file)
--- a/src/test/rgw/bucket_notification/bntests.conf.multisite
+++ b/src/test/rgw/bucket_notification/bntests.conf.multisite
@@ -11,3 +11,8 @@ secret_key = crayon
 display_name = RegularUser
 user_id = regular.user
 email =
+
+[kerberos]
+service_name = kafka
+principal = rgw/<broker-fqdn/ip>@REALM.COM
+keytab = /etc/krb5-keytabs/rgw.service.keytab
\ No newline at end of file

diff --git a/src/test/rgw/bucket_notification/kafka-security.sh b/src/test/rgw/bucket_notification/kafka-security.sh
index b3d216b0cd2618d6a1a491a6d35644270e7e307b..07e0dd0598095c3df6346b4ae645eb0f85a4fb0c 100755 (executable)
--- a/src/test/rgw/bucket_notification/kafka-security.sh
+++ b/src/test/rgw/bucket_notification/kafka-security.sh
@@ -8,6 +8,9 @@ REQFILE=$FQDN.req
 CERTFILE=$FQDN.crt
 MYPW=mypassword
 VALIDITY=36500
+CLIENT_KEYFILE=client.key
+CLIENT_CERTFILE=client.crt
+CLIENT_REQFILE=client.req
 
 rm -f $KEYFILE
 rm -f $TRUSTFILE
@@ -66,9 +69,6 @@ keytool -storepass $MYPW -keystore $KEYFILE -alias localhost \
   -import -file $CERTFILE
 
 echo "########## generate client certificate for mTLS testing"
-CLIENT_KEYFILE=client.key
-CLIENT_CERTFILE=client.crt
-CLIENT_REQFILE=client.req
 
 # generate client private key (PKCS#8 for compatibility)
 openssl genpkey -algorithm RSA -out $CLIENT_KEYFILE -pkeyopt rsa_keygen_bits:2048

diff --git a/src/test/rgw/bucket_notification/requirements.txt b/src/test/rgw/bucket_notification/requirements.txt
index 55685e4842d03f73fc2c159560da7a7bc4f596c8..8c1d0ad8807eb3a8503f593a27c478ad73db848a 100644 (file)
--- a/src/test/rgw/bucket_notification/requirements.txt
+++ b/src/test/rgw/bucket_notification/requirements.txt
@@ -6,3 +6,4 @@ pika
 cloudevents>=1.12.1,<2
 xmltodict
 requests>=2.28.0
+gssapi>=1.8.0

diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index fb941e7185bb3d8f2917e1630672a17207ec8ead..0f22b4ffb4d154a96f9cd21deab53ff2161cc9a1 100644 (file)
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -30,7 +30,8 @@ from . import(
     get_config_zonegroup,
     get_config_cluster,
     get_access_key,
-    get_secret_key
+    get_secret_key,
+    get_kerberos_config,
     )
 
 from .api import PSTopicS3, \
@@ -426,6 +427,12 @@ default_kafka_server = get_ip()
 KAFKA_TEST_USER = 'alice'
 KAFKA_TEST_PASSWORD = 'alice-secret'
 
+
+def get_kerberos_env():
+    service_name, principal, keytab = get_kerberos_config()
+    return service_name, principal, keytab
+
+
 def setup_scram_users_via_kafka_configs(mechanism: str) -> None:
     """to setup SCRAM users using kafka-configs.sh after Kafka is running."""
     if not mechanism.startswith('SCRAM'):
@@ -552,17 +559,27 @@ class KafkaReceiver(object):
             if ca_cert:
                 base_config['ssl_cafile'] = ca_cert
             base_config['sasl_mechanism'] = mechanism
-            base_config.update({
-                'sasl_plain_username': KAFKA_TEST_USER,
-                'sasl_plain_password': KAFKA_TEST_PASSWORD,
-            })
+            if mechanism == 'GSSAPI':
+                kerberos_service_name, _, _ = get_kerberos_env()
+                if kerberos_service_name:
+                    base_config['sasl_kerberos_service_name'] = kerberos_service_name
+            else:
+                base_config.update({
+                    'sasl_plain_username': KAFKA_TEST_USER,
+                    'sasl_plain_password': KAFKA_TEST_PASSWORD,
+                })
         elif effective_protocol == 'SASL_PLAINTEXT':
             base_config['security_protocol'] = 'SASL_PLAINTEXT'
             base_config['sasl_mechanism'] = mechanism
-            base_config.update({
-                'sasl_plain_username': KAFKA_TEST_USER,
-                'sasl_plain_password': KAFKA_TEST_PASSWORD,
-            })
+            if mechanism == 'GSSAPI':
+                kerberos_service_name, _, _ = get_kerberos_env()
+                if kerberos_service_name:
+                    base_config['sasl_kerberos_service_name'] = kerberos_service_name
+            else:
+                base_config.update({
+                    'sasl_plain_username': KAFKA_TEST_USER,
+                    'sasl_plain_password': KAFKA_TEST_PASSWORD,
+                })
 
         remaining_retries = 10
         while remaining_retries > 0:
@@ -4732,6 +4749,19 @@ def kafka_security(security_type, mechanism='PLAIN', use_topic_attrs_for_creds=F
     if mechanism.startswith('SCRAM'):
         setup_scram_users_via_kafka_configs(mechanism)
         time.sleep(2)  # Allow time for SCRAM config to propagate
+    elif mechanism == 'GSSAPI':
+        service_name, principal, keytab = get_kerberos_env()
+        missing = []
+        if not service_name:
+            missing.append('service_name')
+        if not principal:
+            missing.append('principal')
+        if not keytab:
+            missing.append('keytab')
+        if missing:
+            pytest.skip('Missing GSSAPI options in [kerberos] section of BNTESTS_CONF: ' + ', '.join(missing))
+        if not os.path.isfile(keytab):
+            pytest.skip(f'[kerberos] keytab does not exist: {keytab}')
     
     conn = connection()
     zonegroup = get_config_zonegroup()
@@ -4742,26 +4772,45 @@ def kafka_security(security_type, mechanism='PLAIN', use_topic_attrs_for_creds=F
     topic_name = bucket_name+'_topic'
     # create topic
     if security_type == 'SASL_SSL':
-        if not use_topic_attrs_for_creds:
-            endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9094'
-        else:
+        if mechanism == 'GSSAPI' or use_topic_attrs_for_creds:
             endpoint_address = 'kafka://' + default_kafka_server + ':9094'
+        else:
+            endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9094'
     elif security_type == 'SSL':
         if use_mtls:
             endpoint_address = 'kafka://' + default_kafka_server + ':9096'
         else:
             endpoint_address = 'kafka://' + default_kafka_server + ':9093'
     elif security_type == 'SASL_PLAINTEXT':
-        endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9095'
+        if mechanism == 'GSSAPI':
+            endpoint_address = 'kafka://' + default_kafka_server + ':9095'
+        else:
+            endpoint_address = 'kafka://alice:alice-secret@' + default_kafka_server + ':9095'
     else:
         assert False, 'unknown security method '+security_type
 
     if security_type == 'SASL_PLAINTEXT':
         endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker&use-ssl=false&mechanism='+mechanism
+        if mechanism == 'GSSAPI':
+            kerberos_service_name, kerberos_principal, kerberos_keytab = get_kerberos_env()
+            if kerberos_service_name:
+                endpoint_args += '&sasl-kerberos-service-name=' + kerberos_service_name
+            if kerberos_principal:
+                endpoint_args += '&sasl-kerberos-principal=' + kerberos_principal
+            if kerberos_keytab:
+                endpoint_args += '&sasl-kerberos-keytab=' + kerberos_keytab
     elif security_type == 'SASL_SSL':
         kafka_cert_dir = _kafka_cert_dir()
         endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker&use-ssl=true&ca-location='+kafka_cert_dir+'/y-ca.crt&mechanism='+mechanism
-        if use_topic_attrs_for_creds:
+        if mechanism == 'GSSAPI':
+            kerberos_service_name, kerberos_principal, kerberos_keytab = get_kerberos_env()
+            if kerberos_service_name:
+                endpoint_args += '&sasl-kerberos-service-name=' + kerberos_service_name
+            if kerberos_principal:
+                endpoint_args += '&sasl-kerberos-principal=' + kerberos_principal
+            if kerberos_keytab:
+                endpoint_args += '&sasl-kerberos-keytab=' + kerberos_keytab
+        elif use_topic_attrs_for_creds:
             endpoint_args += '&user-name=alice&password=alice-secret'
     else:
         kafka_cert_dir = _kafka_cert_dir()
@@ -4886,6 +4935,16 @@ def test_notification_kafka_security_ssl_sasl_scram_512():
     kafka_security('SASL_SSL', mechanism='SCRAM-SHA-512')
 
 
+@pytest.mark.kafka_security_test
+def test_notification_kafka_security_sasl_gssapi():
+    kafka_security('SASL_PLAINTEXT', mechanism='GSSAPI')
+
+
+@pytest.mark.kafka_security_test
+def test_notification_kafka_security_ssl_sasl_gssapi():
+    kafka_security('SASL_SSL', mechanism='GSSAPI')
+
+
 @pytest.mark.kafka_security_test
 def test_notification_kafka_security_ssl_mtls():
     """test mTLS client certificate authentication to Kafka"""