SERVER-75989 Add support for OpenSSL 3.0 FIPS

1 files changed, 47 insertions, 10 deletions

diff --git a/src/mongo/util/net/openssl_init.cpp b/src/mongo/util/net/openssl_init.cpp
index 827fe7833a4..e58f5d027e2 100644
--- a/src/mongo/util/net/openssl_init.cpp
+++ b/src/mongo/util/net/openssl_init.cpp
@@ -45,6 +45,10 @@
 #include <stack>
 #include <vector>
 
+#if OPENSSL_VERSION_NUMBER > 0x30000000L
+#include <openssl/provider.h>
+#endif
+
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kNetwork
 
 
@@ -146,21 +150,54 @@ private:
     }
 };
 
-void setupFIPS() {
-// Turn on FIPS mode if requested, OPENSSL_FIPS must be defined by the OpenSSL headers
-#if defined(MONGO_CONFIG_HAVE_FIPS_MODE_SET)
+#if OPENSSL_VERSION_NUMBER > 0x30000000L
+#define _SUPPORT_FIPS 1
+
+OSSL_PROVIDER* fipsProvider;
+OSSL_PROVIDER* baseProvider;
+
+void initFIPS() {
+    // OpenSSL 3 has a different FIPS design then previous OpenSSL. To load FIPS, we use the FIPS
+    // algorithm provider which we load into the "default" library context.
+    fipsProvider = OSSL_PROVIDER_load(NULL, "fips");
+    if (fipsProvider == NULL) {
+        LOGV2_FATAL_NOTRACE(
+            7585801,
+            "Failed to load OpenSSL 3 FIPS provider. OpenSSL was not compiled with FIPS support.",
+            "error"_attr = SSLManagerInterface::getSSLErrorMessage(ERR_get_error()));
+    }
+
+    // Base provide has non-cryptographic algorihms (like encoding/decoding keys)
+    baseProvider = OSSL_PROVIDER_load(NULL, "base");
+    if (baseProvider == NULL) {
+        LOGV2_FATAL_NOTRACE(7585802,
+                            "Failed to load OpenSSL 3 Base provider",
+                            "error"_attr =
+                                SSLManagerInterface::getSSLErrorMessage(ERR_get_error()));
+    }
+}
+#elif defined(MONGO_CONFIG_HAVE_FIPS_MODE_SET)
+
+#define _SUPPORT_FIPS 1
+
+void initFIPS() {
     int status = FIPS_mode_set(1);
     if (!status) {
-        LOGV2_FATAL(23173,
-                    "can't activate FIPS mode: {error}",
-                    "Can't activate FIPS mode",
-                    "error"_attr = SSLManagerInterface::getSSLErrorMessage(ERR_get_error()));
-        fassertFailedNoTrace(16703);
+        LOGV2_FATAL_NOTRACE(23173,
+                            "Can't activate FIPS mode",
+                            "error"_attr =
+                                SSLManagerInterface::getSSLErrorMessage(ERR_get_error()));
     }
+}
+#endif
+
+void setupFIPS() {
+// Turn on FIPS mode if requested, OPENSSL_FIPS must be defined by the OpenSSL headers
+#if defined(_SUPPORT_FIPS)
+    initFIPS();
     LOGV2(23172, "FIPS 140-2 mode activated");
 #else
-    LOGV2_FATAL(23174, "this version of mongodb was not compiled with FIPS support");
-    fassertFailedNoTrace(17089);
+    LOGV2_FATAL_NOTRACE(23174, "this version of mongodb was not compiled with FIPS support");
 #endif
 }