rgw: under fips & openssl 3.x allow md5 iusage in select rgw ops

openssl 3.x (ex:RHEL9) requires a different override mechanism for MD5 usage under FIPS
for non-cryptographic putposes than openssl 1.x (RHEL8)

fixes: https://tracker.ceph.com/issues/58332

Signed-off-by: Mark Kogan <mkogan@redhat.com>

diff --git a/src/common/ceph_crypto.cc b/src/common/ceph_crypto.cc
index d658edd6af1296328716821f24da804934dd6ecc..18e655b937ac16492d418185ceb37bf593c4a762 100644 (file)
--- a/src/common/ceph_crypto.cc
+++ b/src/common/ceph_crypto.cc
@@ -196,14 +196,29 @@ ssl::OpenSSLDigest::OpenSSLDigest(const EVP_MD * _type)
 
 ssl::OpenSSLDigest::~OpenSSLDigest() {
   EVP_MD_CTX_destroy(mpContext);
+  if (mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_MD_free(mpType_FIPS);
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  }
 }
 
 void ssl::OpenSSLDigest::Restart() {
-  EVP_DigestInit_ex(mpContext, mpType, NULL);
+  if (mpType_FIPS) {
+    EVP_DigestInit_ex(mpContext, mpType_FIPS, NULL);
+  } else {
+    EVP_DigestInit_ex(mpContext, mpType, NULL);
+  }
 }
 
 void ssl::OpenSSLDigest::SetFlags(int flags) {
-  EVP_MD_CTX_set_flags(mpContext, flags);
+  if (flags == EVP_MD_CTX_FLAG_NON_FIPS_ALLOW && OpenSSL_version_num() >= 0x30000000L && mpType == EVP_md5() && !mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    mpType_FIPS = EVP_MD_fetch(NULL, "MD5", "fips=no");
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  } else {
+    EVP_MD_CTX_set_flags(mpContext, flags);
+  }
   this->Restart();
 }
 

diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h
index dd1b14ffab69ef10ced6213e82878a4da7b22a44..bcdc0044cbd59685c595b1a7e58e9629af2e862d 100644 (file)
--- a/src/common/ceph_crypto.h
+++ b/src/common/ceph_crypto.h
@@ -54,6 +54,7 @@ namespace TOPNSPC::crypto {
       private:
        EVP_MD_CTX *mpContext;
        const EVP_MD *mpType;
+        EVP_MD *mpType_FIPS = nullptr;
       public:
        OpenSSLDigest (const EVP_MD *_type);
        ~OpenSSLDigest ();