From 0318a086f508ed7f9028d48e8b98e4b2f458a51f Mon Sep 17 00:00:00 2001 From: Marcus Watts Date: Fri, 17 Dec 2021 23:13:09 -0500 Subject: [PATCH] rgw/sse-s3: various improvements. 1. sse-s3 should not require bucketencryption policy, work w/ postobj 2. make bucket key name configurable 3. +rgw_remove_sse_s3_bucket_key 1. for sse-s3 should not require bucketencryption policy, work w/ postobj get_crypt_attribute -> using s->info.crypt_attribute_map instead of s->env to avoid havoing to know about HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM names, crypt_attribute_get -> crypt_attributes.get to consolidate crypt attribute sources rework sse-s3 logic: sse-s3 can be specified entirely in the rest call, so remove requirement that bucket has bucket encryption policy. also avoid term "default encryption", prefer term "test key". 2. for make bucket key name configurable: With this modification, sse-s3 key names default to being the bucket id, but can be configured to instead consist of the owners name, a fixed string, or variations thereof. 3. +rgw_remove_sse_s3_bucket_key For sse-s3, keys are supposed to be managed entirely by s3. This means when a bucket is removed, we should be removing its key, which should no longer be in use for anything. This is only safe if the key was constructed using "%bucket_id", otherwise it might be used in another bucket and we can never remove it automatically. Signed-off-by: Marcus Watts (cherry picked from commit 55daff7ae0aec48ea2210592ebcbaa2d8f397245) --- src/rgw/rgw_crypt.cc | 348 ++++++++++++++++++++++++++++--------------- src/rgw/rgw_crypt.h | 2 + src/rgw/rgw_op.cc | 5 + 3 files changed, 235 insertions(+), 120 deletions(-) diff --git a/src/rgw/rgw_crypt.cc b/src/rgw/rgw_crypt.cc index 0d75749472fbb..22a2e2e3ce18b 100644 --- a/src/rgw/rgw_crypt.cc +++ b/src/rgw/rgw_crypt.cc @@ -16,7 +16,6 @@ #include "crypto/crypto_accel.h" #include "crypto/crypto_plugin.h" #include "rgw/rgw_kms.h" -#include "rgw/rgw_bucket_encryption.h" #include "rapidjson/document.h" #include "rapidjson/writer.h" #include "rapidjson/error/error.h" @@ -859,49 +858,150 @@ typedef enum { X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, X_AMZ_SERVER_SIDE_ENCRYPTION_LAST } crypt_option_e; - -typedef struct { - const char* http_header_name; +struct crypt_option_names { const std::string post_part_name; -} crypt_option_names; +}; static const crypt_option_names crypt_options[] = { - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM", "x-amz-server-side-encryption-customer-algorithm"}, - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY", "x-amz-server-side-encryption-customer-key"}, - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5", "x-amz-server-side-encryption-customer-key-md5"}, - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION", "x-amz-server-side-encryption"}, - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID", "x-amz-server-side-encryption-aws-kms-key-id"}, - {"HTTP_X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT", "x-amz-server-side-encryption-context"}, + { "x-amz-server-side-encryption-customer-algorithm"}, + { "x-amz-server-side-encryption-customer-key"}, + { "x-amz-server-side-encryption-customer-key-md5"}, + { "x-amz-server-side-encryption"}, + { "x-amz-server-side-encryption-aws-kms-key-id"}, + { "x-amz-server-side-encryption-context"}, }; -static std::string_view get_crypt_attribute( - const RGWEnv* env, - std::map* parts, - crypt_option_e option) -{ - static_assert( - X_AMZ_SERVER_SIDE_ENCRYPTION_LAST == sizeof(crypt_options)/sizeof(*crypt_options), - "Missing items in crypt_options"); - if (parts != nullptr) { - auto iter - = parts->find(crypt_options[option].post_part_name); - if (iter == parts->end()) - return std::string_view(); - bufferlist& data = iter->second.data; - std::string_view str = std::string_view(data.c_str(), data.length()); - return rgw_trim_whitespace(str); - } else { - const char* hdr = env->get(crypt_options[option].http_header_name, nullptr); - if (hdr != nullptr) { - return std::string_view(hdr); +struct CryptAttributes { + meta_map_t &x_meta_map; + std::map* parts; + + CryptAttributes(req_state *s, + std::map* _parts) : x_meta_map(s->info.crypt_attribute_map), parts(_parts) { + } + + std::string_view get(crypt_option_e option) + { + static_assert( + X_AMZ_SERVER_SIDE_ENCRYPTION_LAST == sizeof(crypt_options)/sizeof(*crypt_options), + "Missing items in crypt_options"); + if (parts != nullptr) { + auto iter + = parts->find(crypt_options[option].post_part_name); + if (iter == parts->end()) + return std::string_view(); + bufferlist& data = iter->second.data; + std::string_view str = std::string_view(data.c_str(), data.length()); + return rgw_trim_whitespace(str); } else { - return std::string_view(); + auto hdr { x_meta_map.find(crypt_options[option].post_part_name) }; + if (hdr != x_meta_map.end()) { + return std::string_view(hdr->second); + } else { + return std::string_view(); + } } } +}; + +std::string fetch_bucket_key_id(struct req_state *s) +{ + auto kek_iter = s->bucket_attrs.find(RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID); + if (kek_iter == s->bucket_attrs.end()) + return std::string(); + std::string a_key { kek_iter->second.to_str() }; + // early code appends a nul; pretend that didn't happen + auto l { a_key.length() }; + if (l > 0 && a_key[l-1] == '\0') { + a_key.resize(--l); + } + return a_key; } +const std::string cant_expand_key{ "\uFFFD" }; +std::string expand_key_name(struct req_state *s, const std::string_view&t) +{ + std::string r; + size_t i, j; + for (i = 0;;) { + i = t.find('%', (j = i)); + if (i != j) { + if (i == std::string_view::npos) + r.append( t.substr(j) ); + else + r.append( t.substr(j, i-j) ); + } + if (i == std::string_view::npos) { + break; + } + if (t[i+1] == '%') { + r.append("%"); + i += 2; + continue; + } + if (t.compare(i+1, 9, "bucket_id") == 0) { + r.append(s->bucket->get_marker()); + i += 10; + continue; + } + if (t.compare(i+1, 8, "owner_id") == 0) { + r.append(s->bucket->get_info().owner.id); + i += 9; + continue; + } + return cant_expand_key; + } + return r; +} + +static int get_sse_s3_bucket_key(req_state *s, + std::string &key_id) +{ + int res; + std::string saved_key; + + key_id = expand_key_name(s, s->cct->_conf->rgw_crypt_sse_s3_key_template); + + if (key_id == cant_expand_key) { + ldpp_dout(s, 5) << "ERROR: unable to expand key_id " << + s->cct->_conf->rgw_crypt_sse_s3_key_template << " on bucket" << dendl; + s->err.message = "Server side error - unable to expand key_id"; + return -EINVAL; + } + + saved_key = fetch_bucket_key_id(s); + if (saved_key != "") { + ldpp_dout(s, 5) << "Found KEK ID: " << key_id << dendl; + } + if (saved_key != key_id) { + res = create_sse_s3_bucket_key(s, s->cct, key_id); + if (res != 0) { + return res; + } + bufferlist key_id_bl; + key_id_bl.append(key_id.c_str(), key_id.length()); + for (int count = 0; count < 15; ++count) { + rgw::sal::Attrs attrs = s->bucket->get_attrs(); + attrs[RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID] = key_id_bl; + res = s->bucket->merge_and_store_attrs(s, attrs, s->yield); + if (res != -ECANCELED) { + break; + } + res = s->bucket->try_refresh_info(s, nullptr); + if (res != 0) { + break; + } + } + if (res != 0) { + ldpp_dout(s, 5) << "ERROR: unable to save new key_id on bucket" << dendl; + s->err.message = "Server side error - unable to save key_id"; + return res; + } + } + return 0; +} int rgw_s3_prepare_encrypt(struct req_state* s, std::map& attrs, @@ -912,11 +1012,12 @@ int rgw_s3_prepare_encrypt(struct req_state* s, std::map& crypt_http_responses) { int res = 0; + CryptAttributes crypt_attributes { s, parts }; crypt_http_responses.clear(); { std::string_view req_sse_ca = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM); if (! req_sse_ca.empty()) { if (req_sse_ca != "AES256") { ldpp_dout(s, 5) << "ERROR: Invalid value for header " @@ -934,7 +1035,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, std::string key_bin; try { key_bin = from_base64( - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY) ); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY) ); } catch (...) { ldpp_dout(s, 5) << "ERROR: rgw_s3_prepare_encrypt invalid encryption " << "key which contains character that is not base64 encoded." @@ -952,7 +1053,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, } std::string_view keymd5 = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5); std::string keymd5_bin; try { @@ -1000,7 +1101,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, return 0; } else { std::string_view customer_key = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY); if (!customer_key.empty()) { ldpp_dout(s, 5) << "ERROR: SSE-C encryption request is missing the header " << "x-amz-server-side-encryption-customer-algorithm" @@ -1011,7 +1112,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, } std::string_view customer_key_md5 = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5); if (!customer_key_md5.empty()) { ldpp_dout(s, 5) << "ERROR: SSE-C encryption request is missing the header " << "x-amz-server-side-encryption-customer-algorithm" @@ -1024,7 +1125,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, /* AMAZON server side encryption with KMS (key management service) */ std::string_view req_sse = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION); if (! req_sse.empty()) { if (s->cct->_conf->rgw_crypt_require_ssl && @@ -1035,12 +1136,12 @@ int rgw_s3_prepare_encrypt(struct req_state* s, if (req_sse == "aws:kms") { std::string_view context = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT); std::string cooked_context; if ((res = make_canonical_context(s, context, cooked_context))) return res; std::string_view key_id = - get_crypt_attribute(s->info.env, parts, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID); if (key_id.empty()) { ldpp_dout(s, 5) << "ERROR: not provide a valid key id" << dendl; s->err.message = "Server Side Encryption with KMS managed key requires " @@ -1079,7 +1180,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, crypt_http_responses["x-amz-server-side-encryption-context"] = std::move(cooked_context); return 0; } else if (req_sse == "AES256") { - /* if a default encryption key was provided, we will use it for SSE-S3 */ + /* SSE-S3: fall through to logic to look for vault or test key */ } else { ldpp_dout(s, 5) << "ERROR: Invalid value for header x-amz-server-side-encryption" << dendl; @@ -1090,8 +1191,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, } else { /* x-amz-server-side-encryption not present or empty */ std::string_view key_id = - get_crypt_attribute(s->info.env, parts, - X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID); + crypt_attributes.get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID); if (!key_id.empty()) { ldpp_dout(s, 5) << "ERROR: SSE-KMS encryption request is missing the header " << "x-amz-server-side-encryption" @@ -1102,79 +1202,54 @@ int rgw_s3_prepare_encrypt(struct req_state* s, } } - /* Checking bucket attributes if SSE is enabled. Currently only supporting SSE-S3 */ - const auto& buck_attrs = s->bucket_attrs; - auto aiter = buck_attrs.find(RGW_ATTR_BUCKET_ENCRYPTION_POLICY); - if (aiter != buck_attrs.end()) { - ldpp_dout(s, 5) << "Found RGW_ATTR_BUCKET_ENCRYPTION_POLICY on " - << s->bucket_name << dendl; + /* from here on we are only handling SSE-S3 (req_sse=="AES256") */ - bufferlist::const_iterator iter{&aiter->second}; - try { - RGWBucketEncryptionConfig bucket_encryption_conf; - bucket_encryption_conf.decode(iter); - if (bucket_encryption_conf.sse_algorithm() == "AES256") { - ldpp_dout(s, 5) << "RGW_ATTR_BUCKET_ENCRYPTION ALGO: " - << bucket_encryption_conf.sse_algorithm() << dendl; - std::string_view context = ""; - std::string cooked_context; - if ((res = make_canonical_context(s, context, cooked_context))) - return res; - - /* Find the KEK ID */ - auto kek_iter = buck_attrs.find(RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID); - if (kek_iter == buck_attrs.end()) { - ldpp_dout(s, 5) << "ERROR: KEK ID absent for bucket having " - "encryption enabled" << dendl; - s->err.message = "Server side error - SSE-S3 key absent"; - return -EINVAL; - } - - std::string_view key_id = kek_iter->second.to_str(); - ldpp_dout(s, 5) << "Found KEK ID: " << key_id << dendl; - std::string key_selector = create_random_key_selector(s->cct); - - set_attr(attrs, RGW_ATTR_CRYPT_KEYSEL, key_selector); - set_attr(attrs, RGW_ATTR_CRYPT_CONTEXT, cooked_context); - set_attr(attrs, RGW_ATTR_CRYPT_MODE, "AES256"); - set_attr(attrs, RGW_ATTR_CRYPT_KEYID, key_id); - std::string actual_key; - res = make_actual_key_from_sse_s3(s->cct, attrs, actual_key); - if (res != 0) { - ldpp_dout(s, 5) << "ERROR: failed to retrieve actual key from key_id: " << key_id << dendl; - s->err.message = "Failed to retrieve the actual key " ; - return res; - } - if (actual_key.size() != AES_256_KEYSIZE) { - ldpp_dout(s, 5) << "ERROR: key obtained from key_id:" << - key_id << " is not 256 bit size" << dendl; - s->err.message = "SSE-S3 provided an invalid key for the given keyid."; - return -ERR_INVALID_ACCESS_KEY; - } - - if (block_crypt) { - auto aes = std::unique_ptr(new AES_256_CBC(s->cct)); - aes->set_key(reinterpret_cast(actual_key.c_str()), AES_256_KEYSIZE); - *block_crypt = std::move(aes); - } - ::ceph::crypto::zeroize_for_security(actual_key.data(), actual_key.length()); - - crypt_http_responses["x-amz-server-side-encryption"] = "AES256"; - - return 0; - } else { - ldpp_dout(s, 5) << "ERROR: SSE algorithm: " - << bucket_encryption_conf.sse_algorithm() - << " not supported" << dendl; - s->err.message = "The requested encryption algorithm is not valid, must be AES256."; - return -ERR_INVALID_ENCRYPTION_ALGORITHM; - } - } catch (const buffer::error& e) { - ldpp_dout(s, 5) << __func__ << "decode bucket_encryption_conf failed" << dendl; + if (s->cct->_conf->rgw_crypt_sse_s3_backend == "vault") { + ldpp_dout(s, 5) << "RGW_ATTR_BUCKET_ENCRYPTION ALGO: " + << req_sse << dendl; + std::string_view context = ""; + std::string cooked_context; + if ((res = make_canonical_context(s, context, cooked_context))) + return res; + + std::string key_id; + res = get_sse_s3_bucket_key(s, key_id); + if (res != 0) { + return res; + } + std::string key_selector = create_random_key_selector(s->cct); + + set_attr(attrs, RGW_ATTR_CRYPT_KEYSEL, key_selector); + set_attr(attrs, RGW_ATTR_CRYPT_CONTEXT, cooked_context); + set_attr(attrs, RGW_ATTR_CRYPT_MODE, "AES256"); + set_attr(attrs, RGW_ATTR_CRYPT_KEYID, key_id); + std::string actual_key; + res = make_actual_key_from_sse_s3(s, s->cct, attrs, actual_key); + if (res != 0) { + ldpp_dout(s, 5) << "ERROR: failed to retrieve actual key from key_id: " << key_id << dendl; + s->err.message = "Failed to retrieve the actual key"; + return res; + } + if (actual_key.size() != AES_256_KEYSIZE) { + ldpp_dout(s, 5) << "ERROR: key obtained from key_id:" << + key_id << " is not 256 bit size" << dendl; + s->err.message = "SSE-S3 provided an invalid key for the given keyid."; + return -EINVAL; + } + + if (block_crypt) { + auto aes = std::unique_ptr(new AES_256_CBC(s, s->cct)); + aes->set_key(reinterpret_cast(actual_key.c_str()), AES_256_KEYSIZE); + *block_crypt = std::move(aes); } + ::ceph::crypto::zeroize_for_security(actual_key.data(), actual_key.length()); + + crypt_http_responses["x-amz-server-side-encryption"] = "AES256"; + + return 0; } - /* no other encryption mode, check if default encryption is selected */ + /* SSE-S3 and no backend, check if there is a test key */ if (s->cct->_conf->rgw_crypt_default_encryption_key != "") { std::string master_encryption_key; try { @@ -1397,20 +1472,20 @@ int rgw_s3_prepare_decrypt(struct req_state* s, /* try to retrieve actual key */ std::string key_id = get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYID); std::string actual_key; - res = reconstitute_actual_key_from_sse_s3(s->cct, attrs, actual_key); + res = reconstitute_actual_key_from_sse_s3(s, s->cct, attrs, actual_key); if (res != 0) { - ldpp_dout(s, 10) << "ERROR: failed to retrieve actual key " << dendl; - s->err.message = "Failed to retrieve the actual key " ; + ldpp_dout(s, 10) << "ERROR: failed to retrieve actual key" << dendl; + s->err.message = "Failed to retrieve the actual key"; return res; } if (actual_key.size() != AES_256_KEYSIZE) { ldpp_dout(s, 0) << "ERROR: key obtained " << - " is not 256 bit size" << dendl; - s->err.message = "SSE-S3 provided an invalid key for the given keyid."; + "is not 256 bit size" << dendl; + s->err.message = "SSE-S3 provided an invalid key for the given keyid."; return -ERR_INVALID_ACCESS_KEY; } - auto aes = std::unique_ptr(new AES_256_CBC(s->cct)); + auto aes = std::unique_ptr(new AES_256_CBC(s, s->cct)); aes->set_key(reinterpret_cast(actual_key.c_str()), AES_256_KEYSIZE); actual_key.replace(0, actual_key.length(), actual_key.length(), '\000'); if (block_crypt) *block_crypt = std::move(aes); @@ -1424,6 +1499,39 @@ int rgw_s3_prepare_decrypt(struct req_state* s, return 0; } +int rgw_remove_sse_s3_bucket_key(req_state *s) +{ + int res; + auto key_id { expand_key_name(s, s->cct->_conf->rgw_crypt_sse_s3_key_template) }; + auto saved_key { fetch_bucket_key_id(s) }; + size_t i; + + if (key_id == cant_expand_key) { + ldpp_dout(s, 5) << "ERROR: unable to expand key_id " << + s->cct->_conf->rgw_crypt_sse_s3_key_template << " on bucket" << dendl; + s->err.message = "Server side error - unable to expand key_id"; + return -EINVAL; + } + + if (saved_key == "") { + return 0; + } else if (saved_key != key_id) { + ldpp_dout(s, 5) << "Found but will not delete strange KEK ID: " << saved_key << dendl; + return 0; + } + i = s->cct->_conf->rgw_crypt_sse_s3_key_template.find("%bucket_id"); + if (i == std::string_view::npos) { + ldpp_dout(s, 5) << "Kept valid KEK ID: " << saved_key << dendl; + return 0; + } + ldpp_dout(s, 5) << "Removing valid KEK ID: " << saved_key << dendl; + res = remove_sse_s3_bucket_key(s, s->cct, saved_key); + if (res != 0) { + ldpp_dout(s, 0) << "ERROR: Unable to remove KEK ID: " << saved_key << " got " << res << dendl; + } + return res; +} + /********************************************************************* * "BOTTOM OF FILE" * I've left some commented out lines above. They are there for diff --git a/src/rgw/rgw_crypt.h b/src/rgw/rgw_crypt.h index 45d46faffb9a0..8be83e9a003d0 100644 --- a/src/rgw/rgw_crypt.h +++ b/src/rgw/rgw_crypt.h @@ -173,4 +173,6 @@ static inline std::string get_str_attribute(std::map& a return iter->second.to_str(); } +int rgw_remove_sse_s3_bucket_key(req_state *s); + #endif diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index aa434553dad53..f4b01acbfcdca 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3428,6 +3428,11 @@ void RGWDeleteBucket::execute(optional_yield y) return; } + op_ret = rgw_remove_sse_s3_bucket_key(s); + if (op_ret != 0) { + // do nothing; it will already have been logged + } + op_ret = s->bucket->remove_bucket(this, false, false, nullptr, y); if (op_ret < 0 && op_ret == -ECANCELED) { // lost a race, either with mdlog sync or another delete bucket operation. -- 2.39.5