bool _verify_ssl,
const boost::optional<const std::string&>& _ssl_certificate,
const boost::optional<const std::string&>& _ssl_key,
- const boost::optional<const std::string&>& _ssl_key_password
+ const boost::optional<const std::string&>& _ssl_key_password,
+ const boost::optional<const std::string&>& _kerberos_service_name,
+ const boost::optional<const std::string&>& _kerberos_principal,
+ const boost::optional<const std::string&>& _kerberos_keytab
)
: broker(_broker), user(_user), password(_password), ssl(_ssl),
verify_ssl(_verify_ssl) {
if (_ssl_key_password.has_value()) {
ssl_key_password = _ssl_key_password.get();
}
+ if (_kerberos_service_name.has_value()) {
+ kerberos_service_name = _kerberos_service_name.get();
+ }
+ if (_kerberos_principal.has_value()) {
+ kerberos_principal = _kerberos_principal.get();
+ }
+ if (_kerberos_keytab.has_value()) {
+ kerberos_keytab = _kerberos_keytab.get();
+ }
}
// equality operator and hasher functor are needed
lhs.verify_ssl == rhs.verify_ssl &&
lhs.ssl_certificate == rhs.ssl_certificate &&
lhs.ssl_key == rhs.ssl_key &&
- lhs.ssl_key_password == rhs.ssl_key_password;
+ lhs.ssl_key_password == rhs.ssl_key_password &&
+ lhs.kerberos_service_name == rhs.kerberos_service_name &&
+ lhs.kerberos_principal == rhs.kerberos_principal &&
+ lhs.kerberos_keytab == rhs.kerberos_keytab;
}
struct connection_id_hasher {
boost::hash_combine(h, k.ssl_certificate);
boost::hash_combine(h, k.ssl_key);
boost::hash_combine(h, k.ssl_key_password);
+ boost::hash_combine(h, k.kerberos_service_name);
+ boost::hash_combine(h, k.kerberos_principal);
+ boost::hash_combine(h, k.kerberos_keytab);
return h;
}
};
std::string to_string(const connection_id_t& id) {
- return id.broker + ":" + id.user;
+ std::ostringstream ss;
+ ss << id.broker;
+ if (!id.user.empty()) {
+ ss << ":" << id.user;
+ }
+ if (!id.mechanism.empty()) {
+ ss << " mechanism=" << id.mechanism;
+ }
+ if (!id.kerberos_service_name.empty()) {
+ ss << " service=" << id.kerberos_service_name;
+ }
+ if (!id.kerberos_principal.empty()) {
+ ss << " principal=" << id.kerberos_principal;
+ }
+ if (!id.kerberos_keytab.empty()) {
+ ss << " keytab=" << id.kerberos_keytab;
+ }
+ return ss.str();
}
// convert int status to errno - both RGW and librdkafka values
const boost::optional<std::string> ssl_certificate;
const boost::optional<std::string> ssl_key;
const boost::optional<std::string> ssl_key_password;
+ const boost::optional<std::string> kerberos_service_name;
+ const boost::optional<std::string> kerberos_principal;
+ const boost::optional<std::string> kerberos_keytab;
utime_t timestamp = ceph_clock_now();
// cleanup of all internal connection resource
const std::string& _user, const std::string& _password, const boost::optional<const std::string&>& _mechanism,
const boost::optional<const std::string&>& _ssl_certificate,
const boost::optional<const std::string&>& _ssl_key,
- const boost::optional<const std::string&>& _ssl_key_password) :
+ const boost::optional<const std::string&>& _ssl_key_password,
+ const boost::optional<const std::string&>& _kerberos_service_name,
+ const boost::optional<const std::string&>& _kerberos_principal,
+ const boost::optional<const std::string&>& _kerberos_keytab) :
cct(_cct), broker(_broker), use_ssl(_use_ssl), verify_ssl(_verify_ssl), ca_location(_ca_location), user(_user), password(_password), mechanism(_mechanism),
- ssl_certificate(_ssl_certificate), ssl_key(_ssl_key), ssl_key_password(_ssl_key_password) {}
+ ssl_certificate(_ssl_certificate), ssl_key(_ssl_key), ssl_key_password(_ssl_key_password),
+ kerberos_service_name(_kerberos_service_name), kerberos_principal(_kerberos_principal), kerberos_keytab(_kerberos_keytab) {}
// dtor also destroys the internals
~connection_t() {
char errstr[512] = {0};
+ const bool is_gssapi = conn->mechanism && boost::iequals(*conn->mechanism, "GSSAPI");
+
// set message timeout
// according to documentation, value of zero will expire the message based on retries.
// however, testing with librdkafka v1.6.1 did not expire the message in that case. hence, a value of zero is changed to 1ms
// get list of brokers based on the bootstrap broker
if (rd_kafka_conf_set(conf.get(), "bootstrap.servers", conn->broker.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
- if (conn->use_ssl) {
+ if (is_gssapi) {
+ const char* security_protocol = conn->use_ssl ? "SASL_SSL" : "SASL_PLAINTEXT";
+ if (rd_kafka_conf_set(conf.get(), "security.protocol", security_protocol, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
+ rd_kafka_conf_set(conf.get(), "sasl.mechanism", "GSSAPI", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ ldout(conn->cct, 20) << "Kafka connect: configured GSSAPI SASL" << dendl;
+
+ if (conn->kerberos_service_name) {
+ if (rd_kafka_conf_set(conf.get(), "sasl.kerberos.service.name", conn->kerberos_service_name->c_str(), errstr,
+ sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ }
+ if (conn->kerberos_principal) {
+ if (rd_kafka_conf_set(conf.get(), "sasl.kerberos.principal", conn->kerberos_principal->c_str(), errstr,
+ sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ }
+ if (conn->kerberos_keytab) {
+ if (rd_kafka_conf_set(conf.get(), "sasl.kerberos.keytab", conn->kerberos_keytab->c_str(), errstr,
+ sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ } else {
+ ldout(conn->cct, 20) << "Kafka connect: GSSAPI without keytab; using ticket cache" << dendl;
+ }
+ } else if (conn->use_ssl) {
if (!conn->user.empty()) {
// use SSL+SASL
if (rd_kafka_conf_set(conf.get(), "security.protocol", "SASL_SSL", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
- rd_kafka_conf_set(conf.get(), "sasl.username", conn->user.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
- rd_kafka_conf_set(conf.get(), "sasl.password", conn->password.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ rd_kafka_conf_set(conf.get(), "sasl.username", conn->user.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
+ rd_kafka_conf_set(conf.get(), "sasl.password", conn->password.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
ldout(conn->cct, 20) << "Kafka connect: successfully configured SSL+SASL security" << dendl;
if (conn->mechanism) {
ldout(conn->cct, 20) << "Kafka connect: successfully configured security" << dendl;
} else if (!conn->user.empty()) {
- // use SASL+PLAINTEXT
- if (rd_kafka_conf_set(conf.get(), "security.protocol", "SASL_PLAINTEXT", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
- rd_kafka_conf_set(conf.get(), "sasl.username", conn->user.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
- rd_kafka_conf_set(conf.get(), "sasl.password", conn->password.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
- ldout(conn->cct, 20) << "Kafka connect: successfully configured SASL_PLAINTEXT" << dendl;
+ // use SASL+PLAINTEXT
+ if (rd_kafka_conf_set(conf.get(), "security.protocol", "SASL_PLAINTEXT", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
+ rd_kafka_conf_set(conf.get(), "sasl.username", conn->user.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK ||
+ rd_kafka_conf_set(conf.get(), "sasl.password", conn->password.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ ldout(conn->cct, 20) << "Kafka connect: successfully configured SASL_PLAINTEXT" << dendl;
+
+ if (conn->mechanism) {
+ if (rd_kafka_conf_set(conf.get(), "sasl.mechanism", conn->mechanism->c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ ldout(conn->cct, 20) << "Kafka connect: successfully configured SASL mechanism" << dendl;
+ } else {
+ if (rd_kafka_conf_set(conf.get(), "sasl.mechanism", "PLAIN", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ ldout(conn->cct, 20) << "Kafka connect: using default SASL mechanism" << dendl;
+ }
+ }
- if (conn->mechanism) {
- if (rd_kafka_conf_set(conf.get(), "sasl.mechanism", conn->mechanism->c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
- ldout(conn->cct, 20) << "Kafka connect: successfully configured SASL mechanism" << dendl;
- } else {
- if (rd_kafka_conf_set(conf.get(), "sasl.mechanism", "PLAIN", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
- ldout(conn->cct, 20) << "Kafka connect: using default SASL mechanism" << dendl;
- }
+ if (is_gssapi && conn->use_ssl) {
+ if (conn->ca_location) {
+ if (rd_kafka_conf_set(conf.get(), "ssl.ca.location", conn->ca_location->c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
+ ldout(conn->cct, 20) << "Kafka connect: successfully configured CA location" << dendl;
+ } else {
+ ldout(conn->cct, 20) << "Kafka connect: using default CA location" << dendl;
+ }
+ if (rd_kafka_conf_set(conf.get(), "enable.ssl.certificate.verification", conn->verify_ssl ? "true" : "false", errstr,
+ sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
}
// set the global callback for delivery success/fail
boost::optional<const std::string&> brokers,
boost::optional<const std::string&> ssl_certificate,
boost::optional<const std::string&> ssl_key,
- boost::optional<const std::string&> ssl_key_password) {
+ boost::optional<const std::string&> ssl_key_password,
+ boost::optional<const std::string&> topic_kerberos_service_name,
+ boost::optional<const std::string&> topic_kerberos_principal,
+ boost::optional<const std::string&> topic_kerberos_keytab) {
if (stopped) {
ldout(cct, 1) << "Kafka connect: manager is stopped" << dendl;
return false;
password = topic_password.get();
}
- // this should be validated by the regex in parse_url()
- ceph_assert(user.empty() == password.empty());
+ const bool is_gssapi = mechanism.has_value() && boost::iequals(mechanism.get(), "GSSAPI");
- if (!user.empty() && !use_ssl && !g_conf().get_val<bool>("rgw_allow_notification_secrets_in_cleartext")) {
- ldout(cct, 1) << "Kafka connect: user/password are only allowed over secure connection" << dendl;
- return false;
- }
+ if (is_gssapi) {
+ if (!user.empty() || !password.empty()) {
+ ldout(cct, 5) << "Kafka connect: user/password provided with GSSAPI; ignoring" << dendl;
+ }
+ user.clear();
+ password.clear();
+ } else {
+ // this should be validated by the regex in parse_url()
+ ceph_assert(user.empty() == password.empty());
+ if (!user.empty() && !use_ssl && !g_conf().get_val<bool>("rgw_allow_notification_secrets_in_cleartext")) {
+ ldout(cct, 1) << "Kafka connect: user/password are only allowed over secure connection" << dendl;
+ return false;
+ }
+ }
+
// ssl_certificate and ssl_key must both be provided for mTLS
if (ssl_certificate.has_value() != ssl_key.has_value()) {
ldout(cct, 1) << "Kafka connect: both ssl_certificate and ssl_key must be provided for mTLS (got only "
<< (ssl_certificate.has_value() ? "ssl_certificate" : "ssl_key") << ")" << dendl;
return false;
}
+
+ boost::optional<const std::string&> kerberos_service_name;
+ boost::optional<const std::string&> kerberos_principal;
+ boost::optional<const std::string&> kerberos_keytab;
+
+ if (is_gssapi) {
+ if (topic_kerberos_service_name.has_value()) {
+ kerberos_service_name = topic_kerberos_service_name;
+ } else if (!cct->_conf->rgw_kafka_sasl_kerberos_service_name.empty()) {
+ kerberos_service_name = boost::optional<const std::string&>(cct->_conf->rgw_kafka_sasl_kerberos_service_name);
+ }
+ if (topic_kerberos_principal.has_value()) {
+ kerberos_principal = topic_kerberos_principal;
+ }
+ if (topic_kerberos_keytab.has_value()) {
+ kerberos_keytab = topic_kerberos_keytab;
+ }
+ }
+
if (brokers.has_value()) {
broker_list.append(",");
}
connection_id_t tmp_id(broker_list, user, password, ca_location, mechanism,
- use_ssl, verify_ssl, ssl_certificate, ssl_key, ssl_key_password);
+ use_ssl, verify_ssl, ssl_certificate, ssl_key, ssl_key_password,
+ kerberos_service_name, kerberos_principal, kerberos_keytab);
std::lock_guard lock(connections_lock);
const auto it = connections.find(tmp_id);
// note that ssl vs. non-ssl connection to the same host are two separate connections
return false;
}
- auto conn = std::make_unique<connection_t>(cct, broker_list, use_ssl, verify_ssl, ca_location, user, password, mechanism, ssl_certificate, ssl_key, ssl_key_password);
+ auto conn = std::make_unique<connection_t>(cct, broker_list, use_ssl, verify_ssl, ca_location, user, password,
+ mechanism, ssl_certificate, ssl_key, ssl_key_password,
+ kerberos_service_name, kerberos_principal, kerberos_keytab);
if (!new_producer(conn.get())) {
ldout(cct, 10) << "Kafka connect: producer creation failed in new connection" << dendl;
return false;
boost::optional<const std::string&> brokers,
boost::optional<const std::string&> ssl_certificate,
boost::optional<const std::string&> ssl_key,
- boost::optional<const std::string&> ssl_key_password) {
+ boost::optional<const std::string&> ssl_key_password,
+ boost::optional<const std::string&> kerberos_service_name,
+ boost::optional<const std::string&> kerberos_principal,
+ boost::optional<const std::string&> kerberos_keytab) {
std::shared_lock lock(s_manager_mutex);
if (!s_manager) return false;
return s_manager->connect(conn_id, url, use_ssl, verify_ssl, ca_location,
mechanism, user_name, password, brokers,
- ssl_certificate, ssl_key, ssl_key_password);
+ ssl_certificate, ssl_key, ssl_key_password,
+ kerberos_service_name, kerberos_principal, kerberos_keytab);
}
int publish(const connection_id_t& conn_id,