rgw: under fips & openssl 3.x allow md5 iusage in select rgw ops

openssl 3.x (ex:RHEL9) requires a different override mechanism for MD5 usage under FIPS
for non-cryptographic putposes than openssl 1.x (RHEL8)

fixes: https://tracker.ceph.com/issues/58332

Signed-off-by: Mark Kogan <mkogan@redhat.com>
(cherry picked from commit 69cf179746c30396a7a9c8ce0fe3c9194bfb52a3)

diff --git a/src/common/ceph_crypto.cc b/src/common/ceph_crypto.cc
index e1f8705c9e6545f3b2277d0b45c068115e8f4b91..f5450ca68c5e5e239b0445db6342650855b03101 100644 (file)
--- a/src/common/ceph_crypto.cc
+++ b/src/common/ceph_crypto.cc
@@ -190,14 +190,29 @@ ssl::OpenSSLDigest::OpenSSLDigest(const EVP_MD * _type)
 
 ssl::OpenSSLDigest::~OpenSSLDigest() {
   EVP_MD_CTX_destroy(mpContext);
+  if (mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_MD_free(mpType_FIPS);
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  }
 }
 
 void ssl::OpenSSLDigest::Restart() {
-  EVP_DigestInit_ex(mpContext, mpType, NULL);
+  if (mpType_FIPS) {
+    EVP_DigestInit_ex(mpContext, mpType_FIPS, NULL);
+  } else {
+    EVP_DigestInit_ex(mpContext, mpType, NULL);
+  }
 }
 
 void ssl::OpenSSLDigest::SetFlags(int flags) {
-  EVP_MD_CTX_set_flags(mpContext, flags);
+  if (flags == EVP_MD_CTX_FLAG_NON_FIPS_ALLOW && OpenSSL_version_num() >= 0x30000000L && mpType == EVP_md5() && !mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    mpType_FIPS = EVP_MD_fetch(NULL, "MD5", "fips=no");
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  } else {
+    EVP_MD_CTX_set_flags(mpContext, flags);
+  }
   this->Restart();
 }
 

diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h
index 2feced03a99a5a1391489b4eee118bca563a7065..e5f936bb450fb36df8c07144790eb41cdc7504cd 100644 (file)
--- a/src/common/ceph_crypto.h
+++ b/src/common/ceph_crypto.h
@@ -48,6 +48,7 @@ namespace TOPNSPC::crypto {
       private:
        EVP_MD_CTX *mpContext;
        const EVP_MD *mpType;
+        EVP_MD *mpType_FIPS = nullptr;
       public:
        OpenSSLDigest (const EVP_MD *_type);
        ~OpenSSLDigest ();