From: Marcus Watts <mwatts@redhat.com>
Date: Sun, 16 Feb 2020 02:08:29 +0000 (-0500)
Subject: kmip: first pass at implementation logic.
X-Git-Tag: v17.1.0~2721^2~10
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0430ff7f9bee9ba88ba7f8dad1592e4bfbc2cc4d;p=ceph.git

kmip: first pass at implementation logic.

This implements SSE-KMS for the radosgw using kmip.
This uses symmetric raw keys with a name attribute in kmip,
so providing the same functionality as the "kv" key store
in hashicorp vault.

Signed-off-by: Marcus Watts <mwatts@redhat.com>
---

diff --git a/src/common/options.cc b/src/common/options.cc
index 4028ee7bc667..bd53f45144bd 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6998,7 +6998,7 @@ std::vector<Option> get_rgw_options() {
 
     Option("rgw_crypt_s3_kms_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("barbican")
-    .set_enum_allowed({"barbican", "vault", "testing"})
+    .set_enum_allowed({"barbican", "vault", "testing", "kmip"})
     .set_description(
         "Where the SSE-KMS encryption keys are stored. Supported KMS "
         "systems are OpenStack Barbican ('barbican', the default) and HashiCorp "
@@ -7093,8 +7093,8 @@ std::vector<Option> get_rgw_options() {
     .set_description("sse-kms; kmip key names"),
 
     Option("rgw_crypt_kmip_s3_key_template", Option::TYPE_STR, Option::LEVEL_ADVANCED)
-    .set_default("")
-    .set_description("sse-s3; kmip key names"),
+    .set_default("$keyid")
+    .set_description("sse-s3; kmip key template"),
 
     Option("rgw_crypt_suppress_logs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index ab4a239c9758..a62908c63687 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -156,6 +156,7 @@ set(librgw_common_srcs
   rgw_rest_iam.cc
   rgw_object_lock.cc
   rgw_kms.cc
+  rgw_kmip_client.cc
   rgw_url.cc
   rgw_oidc_provider
   rgw_datalog.cc
@@ -180,10 +181,21 @@ target_include_directories(rgw_common PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
 target_include_directories(rgw_common PRIVATE "${LUA_INCLUDE_DIR}")
 
 
+set(librgw_kmip_srcs
+  rgw_kmip_client_impl.cc)
+
+add_library(rgw_kmip OBJECT ${librgw_kmip_srcs})
+
+target_include_directories(rgw_kmip PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip")
+
 target_include_directories(rgw_common PRIVATE
   $<TARGET_PROPERTY:spawn,INTERFACE_INCLUDE_DIRECTORIES>)
 target_compile_definitions(rgw_common PRIVATE
   $<TARGET_PROPERTY:spawn,INTERFACE_COMPILE_DEFINITIONS>)
+target_include_directories(rgw_kmip PRIVATE
+  $<TARGET_PROPERTY:spawn,INTERFACE_INCLUDE_DIRECTORIES>)
+target_compile_definitions(rgw_kmip PRIVATE
+  $<TARGET_PROPERTY:spawn,INTERFACE_COMPILE_DEFINITIONS>)
 
 if(WITH_LTTNG)
   # rgw/rgw_op.cc includes "tracing/rgw_op.h"
@@ -310,6 +322,7 @@ target_link_libraries(rgw_schedulers
   PUBLIC dmclock::dmclock)
 
 add_library(radosgw SHARED ${radosgw_srcs} ${rgw_a_srcs} rgw_main.cc
+  $<TARGET_OBJECTS:rgw_kmip>
   $<TARGET_OBJECTS:civetweb_common_objs>)
 
 add_dependencies(radosgw civetweb_h)
@@ -319,7 +332,7 @@ target_include_directories(radosgw PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/suppo
 target_include_directories(radosgw SYSTEM PUBLIC "../rapidjson/include")
 
 target_link_libraries(radosgw
-  PRIVATE ${rgw_libs} rgw_schedulers
+  PRIVATE ${rgw_libs} rgw_schedulers kmip
   PUBLIC dmclock::dmclock
 )
 if(WITH_RADOSGW_BEAST_FRONTEND)
diff --git a/src/rgw/rgw_kmip_client.cc b/src/rgw/rgw_kmip_client.cc
new file mode 100644
index 000000000000..e801972ea801
--- /dev/null
+++ b/src/rgw/rgw_kmip_client.cc
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "common/Thread.h"
+#include "include/compat.h"
+#include "common/errno.h"
+#include "rgw_common.h"
+#include "rgw_kmip_client.h"
+
+#include <atomic>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+RGWKMIPManager *rgw_kmip_manager;
+
+int
+RGWKMIPTransceiver::wait(optional_yield y)
+{
+  if (done)
+    return ret;
+  std::unique_lock l{lock};
+  if (!done)
+    cond.wait(l);
+  if (ret) {
+    lderr(cct) << "kmip process failed, " << ret << dendl;
+  }
+  return ret;
+}
+
+int
+RGWKMIPTransceiver::send()
+{
+  int r = rgw_kmip_manager->add_request(this);
+  if (r < 0) {
+    lderr(cct) << "kmip send failed, " << r << dendl;
+  }
+  return r;
+}
+
+int
+RGWKMIPTransceiver::process(optional_yield y)
+{
+  int r = send();
+  if (r < 0)
+    return r;
+  return wait(y);
+}
+
+RGWKMIPTransceiver::~RGWKMIPTransceiver()
+{
+  int i;
+  if (out)
+    free(out);
+  out = nullptr;
+  if (outlist->strings) {
+    for (i = 0; i < outlist->string_count; ++i) {
+      free(outlist->strings[i]);
+    }
+    free(outlist->strings);
+    outlist->strings = 0;
+  }
+  if (outkey->data) {
+    ::ceph::crypto::zeroize_for_security(outkey->data, outkey->keylen);
+    free(outkey->data);
+    outkey->data = 0;
+  }
+}
+
+void
+rgw_kmip_client_init(RGWKMIPManager &m)
+{
+  rgw_kmip_manager = &m;
+  rgw_kmip_manager->start();
+}
+
+void
+rgw_kmip_client_cleanup()
+{
+  rgw_kmip_manager->stop();
+  delete rgw_kmip_manager;
+}
diff --git a/src/rgw/rgw_kmip_client.h b/src/rgw/rgw_kmip_client.h
new file mode 100644
index 000000000000..efc7db325b90
--- /dev/null
+++ b/src/rgw/rgw_kmip_client.h
@@ -0,0 +1,67 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#ifndef CEPH_RGW_KMIP_CLIENT_H
+#define CEPH_RGW_KMIP_CLIENT_H
+
+class RGWKMIPManager;
+
+class RGWKMIPTransceiver {
+public:
+  enum kmip_operation {
+    CREATE,
+    LOCATE,
+    GET,
+    GET_ATTRIBUTES,
+    GET_ATTRIBUTE_LIST,
+    DESTROY
+  };
+  CephContext *cct;
+  kmip_operation operation;
+  char *name = 0;
+  char *unique_id = 0;
+  // output - must free
+  char *out = 0;    // unique_id, several
+  struct {    // unique_ids, locate
+    char **strings;
+    int string_count;
+  } outlist[1] = {{0, 0}};
+  struct {    // key, get
+    unsigned char *data;
+    int keylen;
+  } outkey[1] = {0, 0};
+  // end must free
+  int ret;
+  bool done;
+  ceph::mutex lock = ceph::make_mutex("rgw_kmip_req::lock");
+  ceph::condition_variable cond;
+
+  int wait(optional_yield y);
+  RGWKMIPTransceiver(CephContext * const cct,
+    kmip_operation operation)
+  : cct(cct),
+    operation(operation),
+    ret(-EDOM),
+    done(false)
+  {}
+  ~RGWKMIPTransceiver();
+
+  int send();
+  int process(optional_yield y);
+};
+
+class RGWKMIPManager {
+protected:
+  CephContext *cct;
+  bool is_started = false;
+  RGWKMIPManager(CephContext *cct) : cct(cct) {};
+public:
+  virtual ~RGWKMIPManager() { };
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual int add_request(RGWKMIPTransceiver*) = 0;
+};
+
+void rgw_kmip_client_init(RGWKMIPManager &);
+void rgw_kmip_client_cleanup();
+#endif
diff --git a/src/rgw/rgw_kmip_client_impl.cc b/src/rgw/rgw_kmip_client_impl.cc
new file mode 100644
index 000000000000..483738a484c0
--- /dev/null
+++ b/src/rgw/rgw_kmip_client_impl.cc
@@ -0,0 +1,728 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include <boost/intrusive/list.hpp>
+#include <atomic>
+#include <mutex>
+#include <string.h>
+
+#include "include/compat.h"
+#include "common/errno.h"
+#include "rgw_common.h"
+#include "rgw_kmip_client.h"
+#include "rgw_kmip_client_impl.h"
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+extern "C" {
+#include "kmip.h"
+#include "kmip_bio.h"
+#include "kmip_memset.h"
+};
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+static enum kmip_version protocol_version = KMIP_1_0;
+
+struct RGWKmipHandle {
+  int uses;
+  mono_time lastuse;
+  SSL_CTX *ctx;
+  SSL *ssl;
+  BIO *bio;
+  KMIP kmip_ctx[1];
+  TextString textstrings[2];
+  UsernamePasswordCredential upc[1];
+  Credential credential[1];
+  int need_to_free_kmip;
+  size_t buffer_blocks, buffer_block_size, buffer_total_size;
+  uint8 *encoding;
+
+  explicit RGWKmipHandle() :
+    uses(0), ctx(0), ssl(0), bio(0),
+    need_to_free_kmip(0),
+    encoding(0) {
+      memset(kmip_ctx, 0, sizeof kmip_ctx);
+      memset(textstrings, 0, sizeof textstrings);
+      memset(upc, 0, sizeof upc);
+      memset(credential, 0, sizeof credential);
+  };
+};
+
+struct RGWKmipWorker: public Thread {
+  RGWKMIPManagerImpl &m;
+  RGWKmipWorker(RGWKMIPManagerImpl& m) : m(m) {}
+  void *entry() override;
+  void signal() {
+    std::lock_guard l{m.lock};
+    m.cond.notify_all();
+  }
+};
+
+static void
+kmip_free_handle_stuff(RGWKmipHandle *kmip)
+{
+  if (kmip->encoding) {
+    kmip_free_buffer(kmip->kmip_ctx,
+      kmip->encoding,
+      kmip->buffer_total_size);
+    kmip_set_buffer(kmip->kmip_ctx, NULL, 0);
+  }
+  if (kmip->need_to_free_kmip)
+    kmip_destroy(kmip->kmip_ctx);
+  if (kmip->bio)
+    BIO_free_all(kmip->bio);
+  if (kmip->ctx)
+    SSL_CTX_free(kmip->ctx);
+}
+
+class RGWKmipHandleBuilder {
+private:
+  CephContext *cct;
+  const char *clientcert = 0;
+  const char *clientkey = 0;
+  const char *capath = 0;
+  const char *host = 0;
+  const char *portstring = 0;
+  const char *username = 0;
+  const char *password = 0;
+public:
+  RGWKmipHandleBuilder(CephContext *cct) : cct(cct) {};
+  RGWKmipHandleBuilder& set_clientcert(const std::string &v) {
+    const char *s = v.c_str();
+    if (*s) {
+      clientcert = s;
+    }
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_clientkey(const std::string &v) {
+    const char *s = v.c_str();
+    if (*s) {
+      clientkey = s;
+    }
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_capath(const std::string &v) {
+    const char *s = v.c_str();
+    if (*s) {
+      capath = s;
+    }
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_host(const char *v) {
+    host = v;
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_portstring(const char *v) {
+    portstring = v;
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_username(const std::string &v) {
+    const char *s = v.c_str();
+    if (*s) {
+      username = s;
+    }
+    return *this;
+  }
+  RGWKmipHandleBuilder& set_password(const std::string& v) {
+    const char *s = v.c_str();
+    if (*s) {
+      password = s;
+    }
+    return *this;
+  }
+  RGWKmipHandle *build() const;
+};
+
+static int
+kmip_write_an_error_helper(const char *s, size_t l, void *u) {
+  CephContext *cct = (CephContext *)u;
+  std::string es(s, l);
+  lderr(cct) << es << dendl;
+  return l;
+}
+
+void
+ERR_print_errors_ceph(CephContext *cct)
+{
+  ERR_print_errors_cb(kmip_write_an_error_helper, cct);
+}
+
+RGWKmipHandle *
+RGWKmipHandleBuilder::build() const
+{
+  int failed = 1;
+  RGWKmipHandle *r = new RGWKmipHandle();
+  TextString *up = 0;
+	size_t ns;
+
+  r->ctx = SSL_CTX_new(TLS_client_method());
+
+  if (!clientcert)
+    ;
+  else if (SSL_CTX_use_certificate_file(r->ctx, clientcert, SSL_FILETYPE_PEM) != 1) {
+    lderr(cct) << "ERROR: can't load client cert from "
+      << clientcert << dendl;
+    ERR_print_errors_ceph(cct);
+    goto Done;
+  }
+
+  if (!clientkey)
+    ;
+  else if (SSL_CTX_use_PrivateKey_file(r->ctx, clientkey,
+      SSL_FILETYPE_PEM) != 1) {
+    lderr(cct) << "ERROR: can't load client key from "
+      << clientkey << dendl;
+    ERR_print_errors_ceph(cct);
+    goto Done;
+  }
+
+  if (!capath)
+    ;
+  else if (SSL_CTX_load_verify_locations(r->ctx, capath, NULL) != 1) {
+    lderr(cct) << "ERROR: can't load cacert from "
+      << capath << dendl;
+    ERR_print_errors_ceph(cct);
+    goto Done;
+  }
+  r->bio = BIO_new_ssl_connect(r->ctx);
+  if (!r->bio) {
+    lderr(cct) << "BIO_new_ssl_connect failed" << dendl;
+    goto Done;
+  }
+  BIO_get_ssl(r->bio, &r->ssl);
+  SSL_set_mode(r->ssl, SSL_MODE_AUTO_RETRY);
+
+  BIO_set_conn_hostname(r->bio, host);
+  BIO_set_conn_port(r->bio, portstring);
+  if (BIO_do_connect(r->bio) != 1) {
+    lderr(cct) << "BIO_do_connect failed to " << host
+      << ":" << portstring << dendl;
+    ERR_print_errors_ceph(cct);
+    goto Done;
+  }
+
+  // setup kmip
+
+  kmip_init(r->kmip_ctx, NULL, 0, protocol_version);
+	r->need_to_free_kmip = 1;
+	r->buffer_blocks = 1;
+	r->buffer_block_size = 1024;
+	r->encoding = static_cast<uint8*>(r->kmip_ctx->calloc_func(
+    r->kmip_ctx->state, r->buffer_blocks, r->buffer_block_size));
+	if (!r->encoding) {
+		lderr(cct) << "kmip buffer alloc failed: "
+      << r->buffer_blocks <<
+      " * " << r->buffer_block_size << dendl;
+		goto Done;
+	}
+	ns = r->buffer_blocks * r->buffer_block_size;
+	kmip_set_buffer(r->kmip_ctx, r->encoding, ns);
+	r->buffer_total_size = ns;
+
+  up = r->textstrings;
+  if (username) {
+    memset(r->upc, 0, sizeof *r->upc);
+    up->value = (char *) username;
+    up->size = strlen(username);
+    r->upc->username = up++;
+    if (password) {
+      up->value = (char *) password;
+      up->size = strlen(password);
+      r->upc->password = up++;
+    }
+    r->credential->credential_type = KMIP_CRED_USERNAME_AND_PASSWORD;
+    r->credential->credential_value = r->upc;
+    int i = kmip_add_credential(r->kmip_ctx, r->credential);
+    if (i != KMIP_OK) {
+      fprintf(stderr,"failed to add credential to kmip\n");
+      goto Done;
+    }
+  }
+
+  failed = 0;
+Done:
+  if (!failed)
+    ;
+  else if (!r)
+    ;
+  else {
+    kmip_free_handle_stuff(r);
+    delete r;
+    r = 0;
+  }
+  return r;
+}
+
+struct RGWKmipHandles : public Thread {
+  CephContext *cct;
+  ceph::mutex cleaner_lock = ceph::make_mutex("RGWKmipHandles::cleaner_lock");
+  std::vector<RGWKmipHandle*> saved_kmip;
+  int cleaner_shutdown;
+  bool cleaner_active = false;
+  ceph::condition_variable cleaner_cond;
+  RGWKmipHandles(CephContext *cct) :
+    cct(cct), cleaner_shutdown{0} {
+  }
+  RGWKmipHandle* get_kmip_handle();
+  void release_kmip_handle_now(RGWKmipHandle* kmip);
+  void release_kmip_handle(RGWKmipHandle* kmip);
+  void flush_kmip_handles();
+  int do_one_entry(RGWKMIPTransceiver &element);
+  void* entry();
+  void start();
+  void stop();
+};
+
+RGWKmipHandle*
+RGWKmipHandles::get_kmip_handle()
+{
+  RGWKmipHandle* kmip = 0;
+  const char *hostaddr = cct->_conf->rgw_crypt_kmip_addr.c_str();
+  {
+    std::lock_guard lock{cleaner_lock};
+    if (!saved_kmip.empty()) {
+      kmip = *saved_kmip.begin();
+      saved_kmip.erase(saved_kmip.begin());
+    }
+  }
+  if (!kmip && hostaddr) {
+    char *hosttemp = strdup(hostaddr);
+    char *port = strchr(hosttemp, ':');
+    if (port)
+      *port++ = 0;
+    kmip = RGWKmipHandleBuilder{cct}
+      .set_clientcert(cct->_conf->rgw_crypt_kmip_client_cert)
+      .set_clientkey(cct->_conf->rgw_crypt_kmip_client_key)
+      .set_capath(cct->_conf->rgw_crypt_kmip_ca_path)
+      .set_host(hosttemp)
+      .set_portstring(port ? port : "5696")
+      .set_username(cct->_conf->rgw_crypt_kmip_username)
+      .set_password(cct->_conf->rgw_crypt_kmip_password)
+      .build();
+    free(hosttemp);
+  }
+  return kmip;
+}
+
+void
+RGWKmipHandles::release_kmip_handle_now(RGWKmipHandle* kmip)
+{
+  kmip_free_handle_stuff(kmip);
+  delete kmip;
+}
+
+#define MAXIDLE 5
+void
+RGWKmipHandles::release_kmip_handle(RGWKmipHandle* kmip)
+{
+  if (cleaner_shutdown) {
+    release_kmip_handle_now(kmip);
+  } else {
+    std::lock_guard lock{cleaner_lock};
+    kmip->lastuse = mono_clock::now();
+    saved_kmip.insert(saved_kmip.begin(), 1, kmip);
+  }
+}
+
+void*
+RGWKmipHandles::entry()
+{
+  RGWKmipHandle* kmip;
+  std::unique_lock lock{cleaner_lock};
+
+  for (;;) {
+    if (cleaner_shutdown) {
+      if (saved_kmip.empty())
+	break;
+    } else {
+      cleaner_cond.wait_for(lock, std::chrono::seconds(MAXIDLE));
+    }
+    mono_time now = mono_clock::now();
+    while (!saved_kmip.empty()) {
+      auto cend = saved_kmip.end();
+      --cend;
+      kmip = *cend;
+      if (!cleaner_shutdown && now - kmip->lastuse
+	  < std::chrono::seconds(MAXIDLE))
+	break;
+      saved_kmip.erase(cend);
+      release_kmip_handle_now(kmip);
+    }
+  }
+  return nullptr;
+}
+
+void
+RGWKmipHandles::start()
+{
+  std::lock_guard lock{cleaner_lock};
+  if (!cleaner_active) {
+    cleaner_active = true;
+    this->create("KMIPcleaner");  // len<16!!!
+  }
+}
+
+void
+RGWKmipHandles::stop()
+{
+  std::unique_lock lock{cleaner_lock};
+  cleaner_shutdown = 1;
+  cleaner_cond.notify_all();
+  if (cleaner_active) {
+    lock.unlock();
+    this->join();
+    cleaner_active = false;
+  }
+}
+
+void
+RGWKmipHandles::flush_kmip_handles()
+{
+  stop();
+  join();
+  if (!saved_kmip.empty()) {
+    ldout(cct, 0) << "ERROR: " << __func__ << " failed final cleanup" << dendl;
+  }
+  saved_kmip.shrink_to_fit();
+}
+
+int
+RGWKMIPManagerImpl::start()
+{
+  if (worker) {
+    lderr(cct) << "kmip worker already started" << dendl;
+    return -1;
+  }
+  worker = new RGWKmipWorker(*this);
+  worker->create("kmip worker");
+  return 0;
+}
+
+void
+RGWKMIPManagerImpl::stop()
+{
+  going_down = true;
+  if (worker) {
+    worker->signal();
+    worker->join();
+    delete worker;
+    worker = 0;
+  }
+}
+
+int
+RGWKMIPManagerImpl::add_request(RGWKMIPTransceiver *req)
+{
+  std::unique_lock l{lock};
+  if (going_down)
+    return -ECANCELED;
+  requests.push_back(*new Request{*req});
+  l.unlock();
+  if (worker)
+    worker->signal();
+  return 0;
+}
+
+int
+RGWKmipHandles::do_one_entry(RGWKMIPTransceiver &element)
+{
+  auto h = get_kmip_handle();
+  std::unique_lock l{element.lock};
+  Attribute a[8], *ap;
+  TextString nvalue[1], uvalue[1];
+  Name nattr[1];
+  enum cryptographic_algorithm alg = KMIP_CRYPTOALG_AES;
+  int32 length = 256;
+  int32 mask = KMIP_CRYPTOMASK_ENCRYPT | KMIP_CRYPTOMASK_DECRYPT;
+  size_t ns;
+  ProtocolVersion pv[1];
+  RequestHeader rh[1];
+  RequestMessage rm[1];
+  Authentication auth[1];
+  ResponseMessage resp_m[1];
+  int i;
+  union {
+    CreateRequestPayload create_req[1];
+    LocateRequestPayload locate_req[1];
+    GetRequestPayload get_req[1];
+    GetAttributeListRequestPayload lsattrs_req[1];
+    GetAttributesRequestPayload getattrs_req[1];
+  } u[1];
+  RequestBatchItem rbi[1];
+  TemplateAttribute ta[1];
+  const char *what = "?";
+  int need_to_free_response = 0;
+  char *response = NULL;
+  int response_size = 0;
+  enum result_status rs;
+  ResponseBatchItem *req;
+
+  if (!h) {
+    element.ret = -ERR_SERVICE_UNAVAILABLE;
+    return element.ret;
+  }
+  memset(a, 0, sizeof *a);
+  for (i = 0; i < (int)(sizeof a/sizeof *a); ++i)
+    kmip_init_attribute(a+i);
+  ap = a;
+  switch(element.operation) {
+  case RGWKMIPTransceiver::CREATE:
+    ap->type = KMIP_ATTR_CRYPTOGRAPHIC_ALGORITHM;
+    ap->value = &alg;
+    ++ap;
+    ap->type = KMIP_ATTR_CRYPTOGRAPHIC_LENGTH;
+    ap->value = &length;
+    ++ap;
+    ap->type = KMIP_ATTR_CRYPTOGRAPHIC_USAGE_MASK;
+    ap->value = &mask;
+    ++ap;
+    break;
+  default:
+    break;
+  }
+  if (element.name) {
+    memset(nvalue, 0, sizeof *nvalue);
+    nvalue->value = element.name;
+    nvalue->size = strlen(element.name);
+    memset(nattr, 0, sizeof *nattr);
+    nattr->value = nvalue;
+    nattr->type = KMIP_NAME_UNINTERPRETED_TEXT_STRING;
+    ap->type = KMIP_ATTR_NAME;
+    ap->value = nattr;
+    ++ap;
+  }
+  if (element.unique_id) {
+    memset(uvalue, 0, sizeof *uvalue);
+    uvalue->value = element.unique_id;
+    uvalue->size = strlen(element.unique_id);
+  }
+  memset(pv, 0, sizeof *pv);
+  memset(rh, 0, sizeof *rh);
+  memset(rm, 0, sizeof *rm);
+  memset(auth, 0, sizeof *auth);
+  memset(resp_m, 0, sizeof *resp_m);
+  kmip_init_protocol_version(pv, h->kmip_ctx->version);
+  kmip_init_request_header(rh);
+  rh->protocol_version = pv;
+  rh->maximum_response_size = h->kmip_ctx->max_message_size;
+  rh->time_stamp = time(NULL);
+  rh->batch_count = 1;
+  memset(rbi, 0, sizeof *rbi);
+  kmip_init_request_batch_item(rbi);
+  memset(u, 0, sizeof *u);
+  rbi->request_payload = u;
+  switch(element.operation) {
+  case RGWKMIPTransceiver::CREATE:
+    memset(ta, 0, sizeof *ta);
+    ta->attributes = a;
+    ta->attribute_count = ap-a;
+    u->create_req->object_type = KMIP_OBJTYPE_SYMMETRIC_KEY;
+    u->create_req->template_attribute = ta;
+    rbi->operation = KMIP_OP_CREATE;
+    what = "create";
+    break;
+  case RGWKMIPTransceiver::GET:
+    if (element.unique_id)
+      u->get_req->unique_identifier = uvalue;
+    rbi->operation = KMIP_OP_GET;
+    what = "get";
+    break;
+  case RGWKMIPTransceiver::LOCATE:
+    if (ap > a) {
+      u->locate_req->attributes = a;
+      u->locate_req->attribute_count = ap - a;
+    }
+    rbi->operation = KMIP_OP_LOCATE;
+    what = "locate";
+    break;
+  case RGWKMIPTransceiver::GET_ATTRIBUTES:
+  case RGWKMIPTransceiver::GET_ATTRIBUTE_LIST:
+  case RGWKMIPTransceiver::DESTROY:
+  default:
+    lderr(cct) << "Missing operation logic op=" << element.operation << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  rm->request_header = rh;
+  rm->batch_items = rbi;
+  rm->batch_count = 1;
+  if (h->kmip_ctx->credential_list) {
+    LinkedListItem *item = h->kmip_ctx->credential_list->head;
+    if (item) {
+      auth->credential = (Credential *)item->data;
+      rh->authentication = auth;
+    }
+  }
+  for (;;) {
+    i = kmip_encode_request_message(h->kmip_ctx, rm);
+    if (i != KMIP_ERROR_BUFFER_FULL) break;
+    h->kmip_ctx->free_func(h->kmip_ctx->state, h->encoding);
+    h->encoding = 0;
+    ++h->buffer_blocks;
+    h->encoding = static_cast<uint8*>(h->kmip_ctx->calloc_func(h->kmip_ctx->state, h->buffer_blocks, h->buffer_block_size));
+    if (!h->encoding) {
+      lderr(cct) << "kmip buffer alloc failed: "
+	<< h->buffer_blocks
+	<< " * " << h->buffer_block_size << dendl;
+      element.ret = -ENOMEM;
+      goto Done;
+    }
+    ns = h->buffer_blocks * h->buffer_block_size;
+    kmip_set_buffer(h->kmip_ctx, h->encoding, ns);
+    h->buffer_total_size = ns;
+  }
+  if (i != KMIP_OK) {
+    lderr(cct) << " Failed to encode " << what
+      << " request; err=" << i
+      << " ctx error message " << h->kmip_ctx->error_message
+      << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  i = kmip_bio_send_request_encoding(h->kmip_ctx, h->bio,
+    (char*)h->encoding,
+    h->kmip_ctx->index - h->kmip_ctx->buffer,
+    &response, &response_size);
+  if (i < 0) {
+    lderr(cct) << "Problem sending request to " << what << " " << i << " context error message " << h->kmip_ctx->error_message << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  kmip_free_buffer(h->kmip_ctx, h->encoding,
+    h->buffer_total_size);
+  h->encoding = 0;
+  kmip_set_buffer(h->kmip_ctx, response, response_size);
+  need_to_free_response = 1;
+  i = kmip_decode_response_message(h->kmip_ctx, resp_m);
+  if (i != KMIP_OK) {
+    lderr(cct) << "Failed to decode " << what << " " << i << " context error message " << h->kmip_ctx->error_message << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  if (resp_m->batch_count != 1) {
+    lderr(cct) << "Failed; weird response count doing " << what << " " << resp_m->batch_count << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  req = resp_m->batch_items;
+  rs = req->result_status;
+  if (rs != KMIP_STATUS_SUCCESS) {
+    lderr(cct) << "Failed; result status not success " << rs << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  if (req->operation != rbi->operation) {
+    lderr(cct) << "Failed; response operation mismatch, got " << req->operation << " expected " << rbi->operation << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  switch(req->operation)
+  {
+  case KMIP_OP_CREATE: {
+      CreateResponsePayload *pld = (CreateResponsePayload *)req->response_payload;
+      element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1));
+      memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size);
+      element.out[pld->unique_identifier->size] = 0;
+    } break;
+  case KMIP_OP_LOCATE: {
+      LocateResponsePayload *pld = (LocateResponsePayload *)req->response_payload;
+      char **list = static_cast<char **>(malloc(sizeof (char*) * (1 + pld->unique_identifiers_count)));
+      for (i = 0; i < pld->unique_identifiers_count; ++i) {
+	list[i] = static_cast<char *>(malloc(pld->unique_identifiers[i].size+1));
+	memcpy(list[i], pld->unique_identifiers[i].value, pld->unique_identifiers[i].size);
+	list[i][pld->unique_identifiers[i].size] = 0;
+      }
+      list[i] = 0;
+      element.outlist->strings = list;
+      element.outlist->string_count = pld->unique_identifiers_count;
+    } break;
+  case KMIP_OP_GET: {
+      GetResponsePayload *pld = (GetResponsePayload *)req->response_payload;
+      element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1));
+      memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size);
+      element.out[pld->unique_identifier->size] = 0;
+      if (pld->object_type != KMIP_OBJTYPE_SYMMETRIC_KEY) {
+	lderr(cct) << "get: expected symmetric key got " << pld->object_type << dendl;
+	element.ret = -EINVAL;
+	goto Done;
+      }
+      KeyBlock *kp = static_cast<SymmetricKey *>(pld->object)->key_block;
+      ByteString *bp;
+      if (kp->key_format_type != KMIP_KEYFORMAT_RAW) {
+	lderr(cct) << "get: expected raw key fromat got  " << kp->key_format_type << dendl;
+	element.ret = -EINVAL;
+	goto Done;
+      }
+      KeyValue *kv = static_cast<KeyValue *>(kp->key_value);
+      bp  = static_cast<ByteString*>(kv->key_material);
+      element.outkey->data = static_cast<unsigned char *>(malloc(bp->size));
+      element.outkey->keylen = bp->size;
+      memcpy(element.outkey->data, bp->value, bp->size);
+    } break;
+  case KMIP_OP_GET_ATTRIBUTES: {
+      GetAttributesResponsePayload *pld = (GetAttributesResponsePayload *)req->response_payload;
+      element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1));
+      memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size);
+      element.out[pld->unique_identifier->size] = 0;
+    } break;
+  case KMIP_OP_GET_ATTRIBUTE_LIST: {
+      GetAttributeListResponsePayload *pld = (GetAttributeListResponsePayload *)req->response_payload;
+      element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1));
+      memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size);
+      element.out[pld->unique_identifier->size] = 0;
+    } break;
+  case KMIP_OP_DESTROY: {
+      DestroyResponsePayload *pld = (DestroyResponsePayload *)req->response_payload;
+      element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1));
+      memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size);
+      element.out[pld->unique_identifier->size] = 0;
+    } break;
+  default:
+    lderr(cct) << "Missing response logic op=" << element.operation << dendl;
+    element.ret = -EINVAL;
+    goto Done;
+  }
+  element.ret = 0;
+Done:
+  if (need_to_free_response)
+    kmip_free_response_message(h->kmip_ctx, resp_m);
+  element.done = true;
+  element.cond.notify_all();
+  release_kmip_handle(h);
+  return element.ret;
+}
+
+void *
+RGWKmipWorker::entry()
+{
+  std::unique_lock entry_lock{m.lock};
+  ldout(m.cct, 10) << __func__ << " start" << dendl;
+  RGWKmipHandles handles{m.cct};
+  handles.start();
+  while (!m.going_down) {
+    if (m.requests.empty()) {
+      m.cond.wait_for(entry_lock, std::chrono::seconds(MAXIDLE));
+      continue;
+    }
+    auto iter = m.requests.begin();
+    auto element = *iter;
+    m.requests.erase(iter);
+    entry_lock.unlock();
+    (void) handles.do_one_entry(element.details);
+    entry_lock.lock();
+  }
+  for (;;) {
+    if (m.requests.empty()) break;
+    auto iter = m.requests.begin();
+    auto element = std::move(*iter);
+    m.requests.erase(iter);
+    element.details.ret = -666;
+    element.details.done = true;
+    element.details.cond.notify_all();
+  }
+  handles.stop();
+  ldout(m.cct, 10) << __func__ << " finish" << dendl;
+  return nullptr;
+}
diff --git a/src/rgw/rgw_kmip_client_impl.h b/src/rgw/rgw_kmip_client_impl.h
new file mode 100644
index 000000000000..841df87f4c33
--- /dev/null
+++ b/src/rgw/rgw_kmip_client_impl.h
@@ -0,0 +1,29 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#ifndef CEPH_RGW_KMIP_CLIENT_IMPL_H
+#define CEPH_RGW_KMIP_CLIENT_IMPL_H
+struct RGWKmipWorker;
+class RGWKMIPManagerImpl: public RGWKMIPManager {
+protected:
+  ceph::mutex lock = ceph::make_mutex("RGWKMIPManager");
+  ceph::condition_variable cond;
+
+  struct Request : boost::intrusive::list_base_hook<> {
+    boost::intrusive::list_member_hook<> req_hook;
+    RGWKMIPTransceiver &details;
+    Request(RGWKMIPTransceiver &details) : details(details) {}
+  };
+  boost::intrusive::list<Request, boost::intrusive::member_hook< Request,
+  boost::intrusive::list_member_hook<>, &Request::req_hook>> requests;
+  bool going_down = false;
+  RGWKmipWorker *worker = 0;
+public:
+  RGWKMIPManagerImpl(CephContext *cct) : RGWKMIPManager(cct) {};
+  int add_request(RGWKMIPTransceiver *);
+  int start();
+  void stop();
+  friend RGWKmipWorker;
+};
+#endif
+
diff --git a/src/rgw/rgw_kms.cc b/src/rgw/rgw_kms.cc
index dfd2b58f7a08..f251f129067d 100644
--- a/src/rgw/rgw_kms.cc
+++ b/src/rgw/rgw_kms.cc
@@ -12,6 +12,7 @@
 #include "rgw/rgw_keystone.h"
 #include "rgw/rgw_b64.h"
 #include "rgw/rgw_kms.h"
+#include "rgw/rgw_kmip_client.h"
 
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_rgw
@@ -250,6 +251,114 @@ public:
 
 };
 
+class KmipSecretEngine;
+class KmipGetTheKey {
+private:
+	CephContext *cct;
+	std::string work;
+	bool failed = false;
+	int ret;
+protected:
+	KmipGetTheKey(CephContext *cct) : cct(cct) {}
+	KmipGetTheKey& keyid_to_keyname(std::string_view key_id);
+	KmipGetTheKey& get_uniqueid_for_keyname();
+	int get_key_for_uniqueid(std::string &);
+	friend KmipSecretEngine;
+};
+
+KmipGetTheKey&
+KmipGetTheKey::keyid_to_keyname(std::string_view key_id)
+{
+	work = cct->_conf->rgw_crypt_kmip_kms_key_template;
+	std::string keyword = "$keyid";
+	std::string replacement = std::string(key_id);
+	size_t pos = 0;
+	if (work.length() == 0) {
+		work = std::move(replacement);
+	} else {
+		while (pos < work.length()) {
+			pos = work.find(keyword, pos);
+			if (pos == std::string::npos) break;
+			work.replace(pos, keyword.length(), replacement);
+			pos += key_id.length();
+		}
+	}
+	return *this;
+}
+
+KmipGetTheKey&
+KmipGetTheKey::get_uniqueid_for_keyname()
+{
+	RGWKMIPTransceiver secret_req(cct, RGWKMIPTransceiver::LOCATE);
+
+	secret_req.name = (char *) work.c_str();	// XXX ugh constness
+	ret = secret_req.process(null_yield);
+	if (ret < 0) {
+		failed = true;
+	} else if (!secret_req.outlist->string_count) {
+		ret = -ENOENT;
+		lderr(cct) << "error: locate returned no results for "
+			<< secret_req.name << dendl;
+		failed = true;
+	} else if (secret_req.outlist->string_count != 1) {
+		ret = -EINVAL;
+		lderr(cct) << "error: locate found "
+			<< secret_req.outlist->string_count
+			<< " results for " << secret_req.name << dendl;
+		failed = true;
+	} else {
+		work = std::string(secret_req.outlist->strings[0]);
+	}
+	return *this;
+}
+
+int
+KmipGetTheKey::get_key_for_uniqueid(std::string& actual_key)
+{
+	if (failed) return ret;
+	RGWKMIPTransceiver secret_req(cct, RGWKMIPTransceiver::GET);
+	secret_req.unique_id = (char *) work.c_str();	// XXX ugh constness.
+	ret = secret_req.process(null_yield);
+	if (ret < 0) {
+		failed = true;
+	} else {
+		actual_key = std::string((char*)(secret_req.outkey->data),
+			secret_req.outkey->keylen);
+	}
+	return ret;
+}
+
+class KmipSecretEngine: public SecretEngine {
+
+protected:
+  CephContext *cct;
+
+  int send_request(std::string_view key_id, JSONParser* parser) override
+  {
+    return -EINVAL;
+  }
+
+  int decode_secret(JSONObj* json_obj, std::string& actual_key){
+    return -EINVAL;
+  }
+
+public:
+
+  KmipSecretEngine(CephContext *cct) {
+    this->cct = cct;
+  }
+
+  int get_key(std::string_view key_id, std::string& actual_key)
+  {
+	int r;
+	r = KmipGetTheKey{cct}
+		.keyid_to_keyname(key_id)
+		.get_uniqueid_for_keyname()
+		.get_key_for_uniqueid(actual_key);
+	return r;
+  }
+};
+
 
 static map<string,string> get_str_map(const string &str) {
   map<string,string> m;
@@ -383,6 +492,23 @@ static int get_actual_key_from_vault(CephContext *cct,
 }
 
 
+static int get_actual_key_from_kmip(CephContext *cct,
+                                     std::string_view key_id,
+                                     std::string& actual_key)
+{
+  std::string secret_engine = RGW_SSE_KMS_KMIP_SE_KV;
+
+  if (RGW_SSE_KMS_KMIP_SE_KV == secret_engine){
+    KmipSecretEngine engine(cct);
+    return engine.get_key(key_id, actual_key);
+  }
+  else{
+    ldout(cct, 0) << "Missing or invalid secret engine" << dendl;
+    return -EINVAL;
+  }
+}
+
+
 int get_actual_key_from_kms(CephContext *cct,
                             std::string_view key_id,
                             std::string_view key_selector,
@@ -400,6 +526,9 @@ int get_actual_key_from_kms(CephContext *cct,
   if (RGW_SSE_KMS_BACKEND_VAULT == kms_backend)
     return get_actual_key_from_vault(cct, key_id, actual_key);
 
+  if (RGW_SSE_KMS_BACKEND_KMIP == kms_backend)
+    return get_actual_key_from_kmip(cct, key_id, actual_key);
+
   if (RGW_SSE_KMS_BACKEND_TESTING == kms_backend)
     return get_actual_key_from_conf(cct, key_id, key_selector, actual_key);
 
diff --git a/src/rgw/rgw_kms.h b/src/rgw/rgw_kms.h
index 92e85aefe093..89b58ba1c5e9 100644
--- a/src/rgw/rgw_kms.h
+++ b/src/rgw/rgw_kms.h
@@ -11,6 +11,7 @@
 static const std::string RGW_SSE_KMS_BACKEND_TESTING = "testing";
 static const std::string RGW_SSE_KMS_BACKEND_BARBICAN = "barbican";
 static const std::string RGW_SSE_KMS_BACKEND_VAULT = "vault";
+static const std::string RGW_SSE_KMS_BACKEND_KMIP = "kmip";
 
 static const std::string RGW_SSE_KMS_VAULT_AUTH_TOKEN = "token";
 static const std::string RGW_SSE_KMS_VAULT_AUTH_AGENT = "agent";
@@ -18,6 +19,8 @@ static const std::string RGW_SSE_KMS_VAULT_AUTH_AGENT = "agent";
 static const std::string RGW_SSE_KMS_VAULT_SE_TRANSIT = "transit";
 static const std::string RGW_SSE_KMS_VAULT_SE_KV = "kv";
 
+static const std::string RGW_SSE_KMS_KMIP_SE_KV = "kv";
+
 /**
  * Retrieves the actual server-side encryption key from a KMS system given a
  * key ID. Currently supported KMS systems are OpenStack Barbican and HashiCorp
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 357f9ac6eb77..a494c2ce1f01 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#include <boost/intrusive/list.hpp>
 #include "common/ceph_argparse.h"
 #include "global/global_init.h"
 #include "global/signal_handler.h"
@@ -38,6 +39,8 @@
 #include "rgw_process.h"
 #include "rgw_frontend.h"
 #include "rgw_http_client_curl.h"
+#include "rgw_kmip_client.h"
+#include "rgw_kmip_client_impl.h"
 #include "rgw_perf_counters.h"
 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
 #include "rgw_amqp.h"
@@ -322,6 +325,7 @@ int radosgw_Main(int argc, const char **argv)
   rgw_init_resolver();
   rgw::curl::setup_curl(fe_map);
   rgw_http_client_init(g_ceph_context);
+  rgw_kmip_client_init(*new RGWKMIPManagerImpl(g_ceph_context));
   
 #if defined(WITH_RADOSGW_FCGI_FRONTEND)
   FCGX_Init();
@@ -683,6 +687,7 @@ int radosgw_Main(int argc, const char **argv)
   rgw_tools_cleanup();
   rgw_shutdown_resolver();
   rgw_http_client_cleanup();
+  rgw_kmip_client_cleanup();
   rgw::curl::cleanup_curl();
   g_conf().remove_observer(&implicit_tenant_context);
 #ifdef WITH_RADOSGW_AMQP_ENDPOINT