assert isinstance(config, dict)
log.info('Installing Kafka...')
+ kerberos = getattr(ctx, 'kerberos', None)
+
# programmatically find a nearby mirror so as not to hammer archive.apache.org
apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \
"grep -o '<strong>[^<]*</strong>' | sed 's/<[^>]*>//g' | head -n 1"
args=['sudo', 'chmod', 'o+rx', '/home/ubuntu'],
)
- broker_conf(ctx, client, kafka_dir)
+ client_kerberos = kerberos.get(client) if kerberos else None
+ broker_conf(ctx, client, kafka_dir, client_kerberos)
try:
yield
)
-def broker_conf(ctx, client, kafka_dir):
+def broker_conf(ctx, client, kafka_dir, kerberos):
"""writing a custom server.properties config file"""
(remote,) = ctx.cluster.only(client).remotes.keys()
ip = remote.ip_address
"listener.name.mtls.ssl.truststore.location={tdir}/server.truststore.jks\n"
"listener.name.mtls.ssl.truststore.password=mypassword\n"
# SASL mechanisms
- "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n"
+ "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI\n"
"sasl.mechanism.inter.broker.protocol=PLAIN\n"
+ "sasl.kerberos.service.name={service_name}\n"
'listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret" \\\n'
'listener.name.sasl_ssl.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret";\n'
+ 'listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+ ' useKeyTab=true \\\n'
+ ' storeKey=true \\\n'
+ ' keyTab="{keytab}" \\\n'
+ ' principal="{principal}";\n'
'listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret" \\\n'
'listener.name.sasl_plaintext.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret";\n'
+ 'listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+ ' useKeyTab=true \\\n'
+ ' storeKey=true \\\n'
+ ' keyTab="{keytab}" \\\n'
+ ' principal="{principal}";\n'
).format(
tdir=kafka_dir,
ip=ip,
sasl_ssl=KAFKA_PORTS['SASL_SSL'],
sasl_plaintext=KAFKA_PORTS['SASL_PLAINTEXT'],
mtls=KAFKA_PORTS['MTLS'],
+ service_name=kerberos['service_name'] if kerberos else '',
+ keytab=kerberos['kafka_keytab'] if kerberos else '',
+ principal=kerberos['kafka_principal'] if kerberos else '',
)
file_name = 'server.properties'
log.info("kafka conf file: %s", file_name)
--- /dev/null
+"""
+Deploy and configure a Kerberos KDC for Teuthology.
+
+This task stands up a single-node MIT Kerberos KDC and creates the service
+principals / keytabs used by the GSSAPI Kafka bucket-notification tests:
+
+ - kafka/<remote-ip>@CEPH.TEST (used by the Kafka broker)
+ - rgw/<remote-ip>@CEPH.TEST (used by RGW and the test consumer)
+
+"""
+import contextlib
+import logging
+
+from teuthology import contextutil
+from teuthology import misc as teuthology
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+REALM = 'CEPH.TEST'
+MASTER_PASSWORD = 'ceph-kdc-master'
+KEYTAB_DIR = '/etc/krb5-keytabs'
+KAFKA_KEYTAB = '{}/kafka.service.keytab'.format(KEYTAB_DIR)
+RGW_KEYTAB = '{}/rgw.service.keytab'.format(KEYTAB_DIR)
+SERVICE_NAME = 'kafka'
+
+
+def _distro_paths(remote):
+ """Return distro-specific KDC paths and service names."""
+ if remote.os.package_type == 'rpm':
+ return {
+ 'kdc_conf_dir': '/var/kerberos/krb5kdc',
+ 'kdc_service': 'krb5kdc',
+ 'admin_service': 'kadmin',
+ }
+ return {
+ 'kdc_conf_dir': '/etc/krb5kdc',
+ 'kdc_service': 'krb5-kdc',
+ 'admin_service': 'krb5-admin-server',
+ }
+
+
+@contextlib.contextmanager
+def install_kerberos(ctx, config):
+ """Install the Kerberos KDC, admin server and GSSAPI SASL plugin."""
+ assert isinstance(config, dict)
+ log.info('Installing Kerberos packages...')
+
+ deb_packages = [
+ 'krb5-kdc', 'krb5-admin-server', 'krb5-user',
+ 'libkrb5-dev', 'libsasl2-modules-gssapi-mit',
+ ]
+ rpm_packages = [
+ 'krb5-devel', 'krb5-server', 'krb5-workstation',
+ 'cyrus-sasl-gssapi',
+ ]
+
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ if remote.os.package_type == 'rpm':
+ remote.run(args=['sudo', 'dnf', 'install', '-y'] + rpm_packages)
+ else:
+ remote.run(args=['sudo', 'apt-get', 'update'])
+ remote.run(args=[
+ 'sudo', 'DEBIAN_FRONTEND=noninteractive',
+ 'apt-get', 'install', '-y',
+ ] + deb_packages)
+
+ try:
+ yield
+ finally:
+ log.info('Removing Kerberos packages...')
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ if remote.os.package_type == 'rpm':
+ remote.run(
+ args=['sudo', 'dnf', 'remove', '-y'] + rpm_packages,
+ check_status=False,
+ )
+ else:
+ remote.run(
+ args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
+ 'apt-get', 'purge', '-y'] + deb_packages,
+ check_status=False,
+ )
+
+
+@contextlib.contextmanager
+def setup_kdc(ctx, config):
+ """Create the realm, principals and keytabs and start the KDC."""
+ assert isinstance(config, dict)
+ log.info('Setting up Kerberos KDC...')
+
+ if not hasattr(ctx, 'kerberos'):
+ ctx.kerberos = {}
+
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ try:
+ ip = remote.ip_address
+ paths = _distro_paths(remote)
+ kafka_principal = 'kafka/{ip}@{realm}'.format(ip=ip, realm=REALM)
+ rgw_principal = 'rgw/{ip}@{realm}'.format(ip=ip, realm=REALM)
+
+ # /etc/krb5.conf: rdns=false keeps the SPN IP-based (no reverse
+ # lookup to a hostname); dns_lookup_kdc/realm=false keeps the KDC
+ # and realm settings local and static, avoiding any DNS-related
+ # issues in the test environment.
+ krb5_conf = (
+ "[libdefaults]\n"
+ " default_realm = {realm}\n"
+ " dns_lookup_kdc = false\n"
+ " dns_lookup_realm = false\n"
+ " rdns = false\n"
+ " udp_preference_limit = 1\n"
+ "\n"
+ "[realms]\n"
+ " {realm} = {{\n"
+ " kdc = {ip}\n"
+ " admin_server = {ip}\n"
+ " }}\n"
+ ).format(realm=REALM, ip=ip)
+ remote.sudo_write_file('/etc/krb5.conf', krb5_conf)
+
+ # kdc.conf
+ kdc_conf = (
+ "[kdcdefaults]\n"
+ " kdc_ports = 88\n"
+ " kdc_tcp_ports = 88\n"
+ "\n"
+ "[realms]\n"
+ " {realm} = {{\n"
+ " max_life = 24h 0m 0s\n"
+ " max_renewable_life = 7d 0h 0m 0s\n"
+ " }}\n"
+ ).format(realm=REALM)
+ remote.run(args=['sudo', 'mkdir', '-p', paths['kdc_conf_dir']])
+ remote.sudo_write_file(
+ '{}/kdc.conf'.format(paths['kdc_conf_dir']), kdc_conf)
+ remote.sudo_write_file(
+ '{}/kadm5.acl'.format(paths['kdc_conf_dir']),
+ '*/admin@{realm} *\n'.format(realm=REALM))
+
+ # create the KDC database
+ remote.run(args=[
+ 'sudo', 'kdb5_util', 'create', '-s', '-r', REALM,
+ '-P', MASTER_PASSWORD,
+ ])
+
+ # create principals and extract keytabs
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'addprinc -randkey {}'.format(kafka_principal),
+ ])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'addprinc -randkey {}'.format(rgw_principal),
+ ])
+ remote.run(args=['sudo', 'mkdir', '-p', KEYTAB_DIR])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'ktadd -k {keytab} {principal}'.format(
+ keytab=KAFKA_KEYTAB, principal=kafka_principal),
+ ])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'ktadd -k {keytab} {principal}'.format(
+ keytab=RGW_KEYTAB, principal=rgw_principal),
+ ])
+ # make the keytabs world-readable to avoid "keytab contains
+ # no suitable keys" permission failures.
+ remote.run(args=[
+ 'sudo', 'chmod', '644', KAFKA_KEYTAB, RGW_KEYTAB,
+ ])
+
+ # restart the KDC and admin server
+ remote.run(args=[
+ 'sudo', 'systemctl', 'restart',
+ paths['kdc_service'], paths['admin_service'],
+ ])
+
+ # kinit with the RGW principal and list the tickets
+ remote.run(args=[
+ 'kinit', '-kt', RGW_KEYTAB, rgw_principal, run.Raw('&&'), 'klist',
+ ])
+
+ ctx.kerberos[client] = {
+ 'realm': REALM,
+ 'service_name': SERVICE_NAME,
+ 'principal': rgw_principal,
+ 'keytab': RGW_KEYTAB,
+ 'kafka_principal': kafka_principal,
+ 'kafka_keytab': KAFKA_KEYTAB,
+ 'ip': ip,
+ }
+ except Exception as exc:
+ log.exception('Kerberos setup failed for %s', client)
+ raise RuntimeError('Kerberos setup is failing') from exc
+
+ try:
+ yield
+ finally:
+ log.info('Tearing down Kerberos KDC...')
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ paths = _distro_paths(remote)
+ remote.run(
+ args=['sudo', 'systemctl', 'stop',
+ paths['kdc_service'], paths['admin_service']],
+ check_status=False,
+ )
+ remote.run(
+ args=['sudo', 'kdb5_util', 'destroy', '-f'],
+ check_status=False,
+ )
+ remote.run(
+ args=['sudo', 'rm', '-rf', KEYTAB_DIR, '/etc/krb5.conf',
+ '/tmp/krb5cc_bntests'],
+ check_status=False,
+ )
+ ctx.kerberos.pop(client, None)
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+ """
+ Set up a Kerberos KDC for the GSSAPI Kafka bucket-notification tests.
+
+ Must run before the kafka and notification-tests tasks::
+
+ tasks:
+ - kerberos:
+ client.0:
+ - kafka:
+ client.0:
+ kafka_version: 3.9.2
+ - notification-tests:
+ client.0:
+ extra_attr: ["kafka_test", "kafka_security_test"]
+ """
+ assert config is None or isinstance(config, list) \
+ or isinstance(config, dict), \
+ "task kerberos only supports a list or dictionary for configuration"
+
+ all_clients = ['client.{id}'.format(id=id_)
+ for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+ if config is None:
+ config = all_clients
+ if isinstance(config, list):
+ config = dict.fromkeys(config)
+
+ log.debug('Kerberos config is %s', config)
+
+ with contextutil.nested(
+ lambda: install_kerberos(ctx=ctx, config=config),
+ lambda: setup_kdc(ctx=ctx, config=config),
+ ):
+ yield
if kafka_dir:
args.append('KAFKA_DIR={kafka_dir}'.format(kafka_dir=kafka_dir))
+ # when a KDC has been set up, point the GSSAPI client (the kafka-python
+ # consumer used by the tests) at the rgw keytab so it can auto-acquire
+ # and auto-renew its kerberos credentials without a separate kinit.
+ kerberos = getattr(ctx, 'kerberos', None)
+ if kerberos and client in kerberos:
+ keytab = kerberos[client]['keytab']
+ args.append('KRB5_CLIENT_KTNAME={keytab}'.format(keytab=keytab))
+ args.append('KRB5CCNAME=FILE:/tmp/krb5cc_bntests')
+
args.extend([
'{tdir}/ceph/src/test/rgw/bucket_notification/virtualenv/bin/python'.format(tdir=testdir),
'-m', 'pytest',
@contextlib.contextmanager
def task(ctx,config):
"""
- To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server. Also you need to pass the
- 'extra_attr' to the notification tests. Following is the way how to run kafka and finally bucket notification tests::
+ To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server and have kerberos setup.
+ Also you need to pass the 'extra_attr' to the notification tests. Following is the way how to run kerberos, kafka, and
+ finally bucket notification tests::
tasks:
+ - kerberos:
+ client.0:
- kafka:
client.0:
kafka_version: 2.6.0
bntests_conf = {}
+ kerberos = getattr(ctx, 'kerberos', None)
+
for client in clients:
endpoint = ctx.rgw.role_endpoints.get(client)
assert endpoint, 'bntests: no rgw endpoint for {}'.format(client)
+ if kerberos and client in kerberos:
+ kerberos_conf = {
+ 'service_name': kerberos[client]['service_name'],
+ 'principal': kerberos[client]['principal'],
+ 'keytab': kerberos[client]['keytab'],
+ }
+ else:
+ kerberos_conf = {'service_name': '', 'principal': '', 'keytab': ''}
+
cluster_name, _, _ = teuthology.split_role(client)
bntests_conf[client] = ConfigObj(
indent_type='',
+ write_empty_values=True,
infile={
'DEFAULT':
{
'cluster':cluster_name,
'version':'v2'
},
- 's3 main':{}
+ 's3 main':{},
+ 'kerberos':kerberos_conf
}
)
projects / ceph.git / commitdiff