Deal with OpenSSL3's changed defaults when parsing PKCS12 certificates

OpenSSL3 doesn't like old certificates, but macOS doesn't like the new
ones. For the cross-platform signature auto test, we need to enable
handling of legacy certificates in OpenSSL3.

Also fixed the process to regenerate the test signatures (for when
macOS' SecurityFramework may catch up in the future)

Change-Id: Ie95ebb0878e4c6c4b96ef45263575bc2135197d0
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Bernd Weimer <bernd.weimer@qt.io>
Reviewed-by: Dominik Holland <dominik.holland@qt.io>
(cherry picked from commit 68170feaeefef6aa9764205bbcb10249b6a4a5c1)
Reviewed-by: Robert Griebl <robert.griebl@qt.io>

5 files changed, 49 insertions, 6 deletions

diff --git a/src/crypto-lib/cryptography.cpp b/src/crypto-lib/cryptography.cpp
index 6e3fb327..74a1b3bb 100644
--- a/src/crypto-lib/cryptography.cpp
+++ b/src/crypto-lib/cryptography.cpp
@@ -67,6 +67,8 @@
 QT_BEGIN_NAMESPACE_AM
 
 Q_GLOBAL_STATIC(QMutex, initMutex)
+static bool openSslInitialized = false;
+static bool loadOpenSsl3LegacyProvider = false;
 
 // clazy:excludeall=non-pod-global-static
 static AM_LIBCRYPTO_FUNCTION(ERR_error_string_n, void(*)(unsigned long, char *, size_t));
@@ -77,6 +79,7 @@ QT_END_NAMESPACE_AM
 
 QT_BEGIN_NAMESPACE_AM
 
+
 QByteArray Cryptography::generateRandomBytes(int size)
 {
     QByteArray result;
@@ -102,17 +105,26 @@ QByteArray Cryptography::generateRandomBytes(int size)
 void Cryptography::initialize()
 {
 #if defined(AM_USE_LIBCRYPTO)
-    static bool openSslInitialized = false;
-
     QMutexLocker locker(initMutex());
     if (!openSslInitialized) {
-        if (!LibCryptoFunctionBase::initialize())
+        if (!LibCryptoFunctionBase::initialize(loadOpenSsl3LegacyProvider))
             qFatal("Could not load libcrypto");
         openSslInitialized = true;
     }
 #endif
 }
 
+void Cryptography::enableOpenSsl3LegacyProvider()
+{
+#if defined(AM_USE_LIBCRYPTO)
+    QMutexLocker locker(initMutex());
+    if (openSslInitialized)
+        qCritical("Cryptography::enableOpenSsl3LegacyProvider() needs to be called before using any other crypto functions.");
+    else
+        loadOpenSsl3LegacyProvider = true;
+#endif
+}
+
 QString Cryptography::errorString(qint64 osCryptoError, const char *errorDescription)
 {
     QString result;
diff --git a/src/crypto-lib/cryptography.h b/src/crypto-lib/cryptography.h
index bc3c8a28..f1357c5a 100644
--- a/src/crypto-lib/cryptography.h
+++ b/src/crypto-lib/cryptography.h
@@ -54,6 +54,8 @@ QByteArray generateRandomBytes(int size);
 
 void initialize();
 
+void enableOpenSsl3LegacyProvider(); // needs to be called before any other crypto functions
+
 QString errorString(qint64 osCryptoError, const char *errorDescription = nullptr);
 
 }
diff --git a/src/crypto-lib/libcryptofunction.cpp b/src/crypto-lib/libcryptofunction.cpp
index 362307cd..207cde86 100644
--- a/src/crypto-lib/libcryptofunction.cpp
+++ b/src/crypto-lib/libcryptofunction.cpp
@@ -49,6 +49,7 @@
 // we want at least openssl 1.0.1 or 1.1.0
 #define AM_MINIMUM_OPENSSL10_VERSION 0x1000100fL
 #define AM_MINIMUM_OPENSSL11_VERSION 0x1010000fL
+#define AM_MINIMUM_OPENSSL30_VERSION 0x3000000fL
 
 QT_BEGIN_NAMESPACE_AM
 
@@ -60,10 +61,15 @@ static AM_LIBCRYPTO_FUNCTION(ERR_load_crypto_strings, void(*)());
 static AM_LIBCRYPTO_FUNCTION(OpenSSL_version_num, unsigned long(*)(), 0);
 static AM_LIBCRYPTO_FUNCTION(OPENSSL_init_crypto, int(*)(uint64_t, void *), 0);
 
+struct OSSL_PROVIDER;
+struct OSSL_LIB_CTX;
+static AM_LIBCRYPTO_FUNCTION(OSSL_PROVIDER_load, OSSL_PROVIDER *(*)(OSSL_LIB_CTX *, const char *), nullptr);
+
 QLibrary *Cryptography::LibCryptoFunctionBase::s_library = nullptr;
 bool Cryptography::LibCryptoFunctionBase::s_isOpenSSL11 = false;
+bool Cryptography::LibCryptoFunctionBase::s_isOpenSSL30 = false;
 
-bool Cryptography::LibCryptoFunctionBase::initialize()
+bool Cryptography::LibCryptoFunctionBase::initialize(bool loadOpenSsl3LegacyProvider)
 {
     if (s_library)
         return true;
@@ -98,6 +104,19 @@ bool Cryptography::LibCryptoFunctionBase::initialize()
         if (version > 0) {
             if (version >= AM_MINIMUM_OPENSSL11_VERSION) {
                 s_isOpenSSL11 = true;
+
+                if (version >= AM_MINIMUM_OPENSSL30_VERSION) {
+                    s_isOpenSSL30 = true;
+
+                    if (loadOpenSsl3LegacyProvider) {
+                        // openSSL 3.0 might need the legacy provider to read old PKCS12 certs
+                        auto legacyLoaded = am_OSSL_PROVIDER_load(nullptr, "legacy");
+                        auto defaultLoaded = am_OSSL_PROVIDER_load(nullptr, "default");
+                        if (!legacyLoaded || !defaultLoaded)
+                            qCritical("Loaded libcrypto version 3, but couldn't load the 'legacy provider' as requested");
+                    }
+                }
+
                 return (am_OPENSSL_init_crypto(4 /*OPENSSL_INIT_ADD_ALL_CIPHERS*/
                                                | 8 /*OPENSSL_INIT_ADD_ALL_DIGESTS*/
                                                | 2 /*OPENSSL_INIT_LOAD_CRYPTO_STRINGS*/,
diff --git a/src/crypto-lib/libcryptofunction.h b/src/crypto-lib/libcryptofunction.h
index d60537d5..86d05674 100644
--- a/src/crypto-lib/libcryptofunction.h
+++ b/src/crypto-lib/libcryptofunction.h
@@ -80,8 +80,9 @@ public:
 class LibCryptoFunctionBase
 {
 public:
-    static bool initialize();
+    static bool initialize(bool loadOpenSsl3LegacyProvider);
     static inline bool isOpenSSL11() { return s_isOpenSSL11; }
+    static inline bool isOpenSSL30() { return s_isOpenSSL30; }
 
 protected:
     LibCryptoFunctionBase(const char *symbol);
@@ -94,6 +95,7 @@ protected:
 private:
     static QLibrary *s_library;
     static bool s_isOpenSSL11;
+    static bool s_isOpenSSL30;
     bool m_tried = false;
 };
 
diff --git a/tests/signature/tst_signature.cpp b/tests/signature/tst_signature.cpp
index 9f48590a..ccf86474 100644
--- a/tests/signature/tst_signature.cpp
+++ b/tests/signature/tst_signature.cpp
@@ -32,6 +32,7 @@
 
 #include "global.h"
 #include "signature.h"
+#include "cryptography.h"
 
 QT_USE_NAMESPACE_AM
 
@@ -55,7 +56,14 @@ private:
 };
 
 tst_Signature::tst_Signature()
-{ }
+{
+    // OpenSSL3 changed a few defaults and it will not accept old PKCS12 certificates
+    // anymore. Regenerating "signing.p12" doesn't help, because the macOS/iOS
+    // SecurityFramework cannot deal with the new algorithms used by OpenSSL3.
+    // The only way out for this cross-platform test case is to enable the so called
+    // "legacy provider" in OpenSSL3 and continue testing with the old certificate.
+    Cryptography::enableOpenSsl3LegacyProvider();
+}
 
 void tst_Signature::initTestCase()
 {