Deal with OpenSSL3's changed defaults when parsing PKCS12 certificates

OpenSSL3 doesn't like old certificates, but macOS doesn't like the new
ones. For the cross-platform signature auto test, we need to enable
handling of legacy certificates in OpenSSL3.

Also fixed the process to regenerate the test signatures (for when
macOS' SecurityFramework may catch up in the future)

Change-Id: Ie95ebb0878e4c6c4b96ef45263575bc2135197d0
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Bernd Weimer <bernd.weimer@qt.io>
Reviewed-by: Dominik Holland <dominik.holland@qt.io>
(cherry picked from commit 68170feaeefef6aa9764205bbcb10249b6a4a5c1)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

7 files changed, 63 insertions, 12 deletions

diff --git a/src/crypto-lib/cryptography.cpp b/src/crypto-lib/cryptography.cpp
index 5c9674a0..3ca407eb 100644
--- a/src/crypto-lib/cryptography.cpp
+++ b/src/crypto-lib/cryptography.cpp
@@ -56,6 +56,8 @@
 QT_BEGIN_NAMESPACE_AM
 
 Q_GLOBAL_STATIC(QMutex, initMutex)
+static bool openSslInitialized = false;
+static bool loadOpenSsl3LegacyProvider = false;
 
 // clazy:excludeall=non-pod-global-static
 static AM_LIBCRYPTO_FUNCTION(ERR_error_string_n, void(*)(unsigned long, char *, size_t));
@@ -66,6 +68,7 @@ QT_END_NAMESPACE_AM
 
 QT_BEGIN_NAMESPACE_AM
 
+
 QByteArray Cryptography::generateRandomBytes(int size)
 {
     QByteArray result;
@@ -91,17 +94,26 @@ QByteArray Cryptography::generateRandomBytes(int size)
 void Cryptography::initialize()
 {
 #if defined(AM_USE_LIBCRYPTO)
-    static bool openSslInitialized = false;
-
     QMutexLocker locker(initMutex());
     if (!openSslInitialized) {
-        if (!LibCryptoFunctionBase::initialize())
+        if (!LibCryptoFunctionBase::initialize(loadOpenSsl3LegacyProvider))
             qFatal("Could not load libcrypto");
         openSslInitialized = true;
     }
 #endif
 }
 
+void Cryptography::enableOpenSsl3LegacyProvider()
+{
+#if defined(AM_USE_LIBCRYPTO)
+    QMutexLocker locker(initMutex());
+    if (openSslInitialized)
+        qCritical("Cryptography::enableOpenSsl3LegacyProvider() needs to be called before using any other crypto functions.");
+    else
+        loadOpenSsl3LegacyProvider = true;
+#endif
+}
+
 QString Cryptography::errorString(qint64 osCryptoError, const char *errorDescription)
 {
     QString result;
diff --git a/src/crypto-lib/cryptography.h b/src/crypto-lib/cryptography.h
index 017fbbaa..48990c59 100644
--- a/src/crypto-lib/cryptography.h
+++ b/src/crypto-lib/cryptography.h
@@ -43,6 +43,8 @@ QByteArray generateRandomBytes(int size);
 
 void initialize();
 
+void enableOpenSsl3LegacyProvider(); // needs to be called before any other crypto functions
+
 QString errorString(qint64 osCryptoError, const char *errorDescription = nullptr);
 
 }
diff --git a/src/crypto-lib/libcryptofunction.cpp b/src/crypto-lib/libcryptofunction.cpp
index 95abbcb8..ff42e803 100644
--- a/src/crypto-lib/libcryptofunction.cpp
+++ b/src/crypto-lib/libcryptofunction.cpp
@@ -38,6 +38,7 @@
 // we want at least openssl 1.0.1 or 1.1.0
 #define AM_MINIMUM_OPENSSL10_VERSION 0x1000100fL
 #define AM_MINIMUM_OPENSSL11_VERSION 0x1010000fL
+#define AM_MINIMUM_OPENSSL30_VERSION 0x3000000fL
 
 QT_BEGIN_NAMESPACE_AM
 
@@ -49,10 +50,15 @@ static AM_LIBCRYPTO_FUNCTION(ERR_load_crypto_strings, void(*)());
 static AM_LIBCRYPTO_FUNCTION(OpenSSL_version_num, unsigned long(*)(), 0);
 static AM_LIBCRYPTO_FUNCTION(OPENSSL_init_crypto, int(*)(uint64_t, void *), 0);
 
+struct OSSL_PROVIDER;
+struct OSSL_LIB_CTX;
+static AM_LIBCRYPTO_FUNCTION(OSSL_PROVIDER_load, OSSL_PROVIDER *(*)(OSSL_LIB_CTX *, const char *), nullptr);
+
 QLibrary *Cryptography::LibCryptoFunctionBase::s_library = nullptr;
 bool Cryptography::LibCryptoFunctionBase::s_isOpenSSL11 = false;
+bool Cryptography::LibCryptoFunctionBase::s_isOpenSSL30 = false;
 
-bool Cryptography::LibCryptoFunctionBase::initialize()
+bool Cryptography::LibCryptoFunctionBase::initialize(bool loadOpenSsl3LegacyProvider)
 {
     if (s_library)
         return true;
@@ -87,6 +93,19 @@ bool Cryptography::LibCryptoFunctionBase::initialize()
         if (version > 0) {
             if (version >= AM_MINIMUM_OPENSSL11_VERSION) {
                 s_isOpenSSL11 = true;
+
+                if (version >= AM_MINIMUM_OPENSSL30_VERSION) {
+                    s_isOpenSSL30 = true;
+
+                    if (loadOpenSsl3LegacyProvider) {
+                        // openSSL 3.0 might need the legacy provider to read old PKCS12 certs
+                        auto legacyLoaded = am_OSSL_PROVIDER_load(nullptr, "legacy");
+                        auto defaultLoaded = am_OSSL_PROVIDER_load(nullptr, "default");
+                        if (!legacyLoaded || !defaultLoaded)
+                            qCritical("Loaded libcrypto version 3, but couldn't load the 'legacy provider' as requested");
+                    }
+                }
+
                 return (am_OPENSSL_init_crypto(4 /*OPENSSL_INIT_ADD_ALL_CIPHERS*/
                                                | 8 /*OPENSSL_INIT_ADD_ALL_DIGESTS*/
                                                | 2 /*OPENSSL_INIT_LOAD_CRYPTO_STRINGS*/,
diff --git a/src/crypto-lib/libcryptofunction.h b/src/crypto-lib/libcryptofunction.h
index 54b8b74d..d85c856d 100644
--- a/src/crypto-lib/libcryptofunction.h
+++ b/src/crypto-lib/libcryptofunction.h
@@ -69,8 +69,9 @@ public:
 class LibCryptoFunctionBase
 {
 public:
-    static bool initialize();
+    static bool initialize(bool loadOpenSsl3LegacyProvider);
     static inline bool isOpenSSL11() { return s_isOpenSSL11; }
+    static inline bool isOpenSSL30() { return s_isOpenSSL30; }
 
 protected:
     LibCryptoFunctionBase(const char *symbol);
@@ -83,6 +84,7 @@ protected:
 private:
     static QLibrary *s_library;
     static bool s_isOpenSSL11;
+    static bool s_isOpenSSL30;
     bool m_tried = false;
 };
 
diff --git a/tests/auto/signature/CMakeLists.txt b/tests/auto/signature/CMakeLists.txt
index fcfbdc5e..f8af0ac1 100644
--- a/tests/auto/signature/CMakeLists.txt
+++ b/tests/auto/signature/CMakeLists.txt
@@ -4,7 +4,7 @@ qt_internal_add_test(tst_signature
         ../error-checking.h
         tst_signature.cpp
     DEFINES
-        AM_TESTDATA_DIR=\\\"${CMAKE_CURRENT_BINARY_DIR}/../../data/\\\"
+        AM_TESTSOURCE_DIR=\\\"${CMAKE_CURRENT_SOURCE_DIR}\\\"
     PUBLIC_LIBRARIES
         Qt::Network
         Qt::AppManCommonPrivate
diff --git a/tests/auto/signature/create-test-data.sh b/tests/auto/signature/create-test-data.sh
index 68f05575..73bf90aa 100644..100755
--- a/tests/auto/signature/create-test-data.sh
+++ b/tests/auto/signature/create-test-data.sh
@@ -33,7 +33,15 @@ echo "Recreating test data"
 
 certdir="../data/certificates/"
 
-[ -f $certdir/dev1.p12 ] || { echo "Please generate test certificates in $certdir first"; exit 1; }
+if [ ! -f $certdir/dev1.p12 ]; then
+  if [ -n "$BUILD_DIR" ] && [ -f "$BUILD_DIR/tests/data/certificates/dev1.p12" ]; then
+    certdir="$BUILD_DIR/tests/data/certificates/"
+  else
+    echo "Please generate test certificates in $certdir first or set"
+    echo "  \$BUILD_DIR to point to your build folder."
+    exit 1
+  fi
+fi
 
 cp $certdir/dev1.p12 signing.p12
 openssl pkcs12 -export -out signing-no-key.p12 -password pass:password -inkey $certdir/dev1-priv.key -nodes -certfile $certdir/ca.crt -in $certdir/dev1.crt -name "Developer 1 Certificate (no key)" -nokeys
diff --git a/tests/auto/signature/tst_signature.cpp b/tests/auto/signature/tst_signature.cpp
index ea4f471d..acdee0a7 100644
--- a/tests/auto/signature/tst_signature.cpp
+++ b/tests/auto/signature/tst_signature.cpp
@@ -33,6 +33,7 @@
 
 #include "global.h"
 #include "signature.h"
+#include "cryptography.h"
 
 QT_USE_NAMESPACE_AM
 
@@ -56,7 +57,14 @@ private:
 };
 
 tst_Signature::tst_Signature()
-{ }
+{
+    // OpenSSL3 changed a few defaults and it will not accept old PKCS12 certificates
+    // anymore. Regenerating "signing.p12" doesn't help, because the macOS/iOS
+    // SecurityFramework cannot deal with the new algorithms used by OpenSSL3.
+    // The only way out for this cross-platform test case is to enable the so called
+    // "legacy provider" in OpenSSL3 and continue testing with the old certificate.
+    Cryptography::enableOpenSsl3LegacyProvider();
+}
 
 void tst_Signature::initTestCase()
 {
@@ -129,15 +137,15 @@ void tst_Signature::crossPlatform()
 
     if (qEnvironmentVariableIsSet("AM_CREATE_SIGNATURE_FILE")) {
         QFile *nativeFile = nullptr;
-#if defined(AM_USE_LIBCRYPTO)
-        nativeFile = &fileOpenSsl;
-#elif defined(Q_OS_WIN)
+#if defined(Q_OS_WIN)
         nativeFile = &fileWinCrypt;
 #elif defined(Q_OS_OSX)
         nativeFile = &fileSecurityFramework;
+#else
+        nativeFile = &fileOpenSsl;
 #endif
         QVERIFY(nativeFile);
-        QFile f(qL1S(AM_TESTDATA_DIR "/../signature") + nativeFile->fileName().mid(1));
+        QFile f(qL1S(AM_TESTSOURCE_DIR "/../signature") + nativeFile->fileName().mid(1));
         QVERIFY2(f.open(QFile::WriteOnly | QFile::Truncate), qPrintable(f.errorString()));
 
         Signature s(hash);