Fix Signature creation on macOS 12

macOS did not support signatures on empty hashes, but macOS 12 now
does. In order to be cross platform compatible, we already prevented
signing those hashes in the OpenSSL backend in the past.
This patch now checks for empty hashes regardless of the backend.

Change-Id: I5c100073cdf9b5329edac495708f4e0288cd952f
Reviewed-by: Bernd Weimer <bernd.weimer@qt.io>
(cherry picked from commit b0c82c2c26d7cbd810ae04899383990aa18052ef)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

3 files changed, 9 insertions, 7 deletions

diff --git a/src/crypto-lib/signature.cpp b/src/crypto-lib/signature.cpp
index 5925b67f..e240f10c 100644
--- a/src/crypto-lib/signature.cpp
+++ b/src/crypto-lib/signature.cpp
@@ -60,6 +60,11 @@ QByteArray Signature::create(const QByteArray &signingCertificatePkcs12, const Q
 {
     d->error.clear();
     try {
+        // Although OpenSSL could, the macOS Security Framework (pre macOS 12) cannot
+        // process empty detached data. So we better just not support it at all.
+        if (d->hash.isEmpty())
+            throw Exception("cannot sign an empty hash value");
+
         QByteArray sig = d->create(signingCertificatePkcs12, signingCertificatePassword);
 //        // very useful while debugging
 //        QFile f(QDir::home().absoluteFilePath("sig.der"));
diff --git a/src/crypto-lib/signature_macos.cpp b/src/crypto-lib/signature_macos.cpp
index be9c781a..0ab19197 100644
--- a/src/crypto-lib/signature_macos.cpp
+++ b/src/crypto-lib/signature_macos.cpp
@@ -55,6 +55,8 @@ public:
     }
 };
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
 
 QByteArray SignaturePrivate::create(const QByteArray &signingCertificatePkcs12,
                                     const QByteArray &signingCertificatePassword) Q_DECL_NOEXCEPT_EXPR(false)
@@ -71,11 +73,8 @@ QByteArray SignaturePrivate::create(const QByteArray &signingCertificatePkcs12,
         QCFString importPassword = QString::fromUtf8(signingCertificatePassword);
         QByteArray keyChainPassword = Cryptography::generateRandomBytes(16);
 
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wdeprecated-declarations"
         // tempnam() is the best thing we can use here, since we cannot supply a file handle
         if ((err = SecKeychainCreate(tempnam(0, 0), 16, keyChainPassword, false, nullptr, &localKeyChain)))
-#pragma clang diagnostic pop
             throw SecurityException(err, "could not create local key-chain");
 
         const void *optionKeys[] = { kSecImportExportPassphrase, kSecImportExportKeychain };
@@ -190,4 +189,6 @@ bool SignaturePrivate::verify(const QByteArray &signaturePkcs7,
     return true;
 }
 
+#pragma clang diagnostic pop
+
 QT_END_NAMESPACE_AM
diff --git a/src/crypto-lib/signature_openssl.cpp b/src/crypto-lib/signature_openssl.cpp
index 97502cd0..ddd51b00 100644
--- a/src/crypto-lib/signature_openssl.cpp
+++ b/src/crypto-lib/signature_openssl.cpp
@@ -121,10 +121,6 @@ public:
 QByteArray SignaturePrivate::create(const QByteArray &signingCertificatePkcs12,
                                     const QByteArray &signingCertificatePassword) Q_DECL_NOEXCEPT_EXPR(false)
 {
-    // Although OpenSSL could, the macOS Security Framework cannot process empty detached data
-    if (hash.isEmpty())
-        throw OpenSslException("cannot sign an empty hash value");
-
     OpenSslPointer<BIO> bioPkcs12(am_BIO_new_mem_buf(signingCertificatePkcs12.constData(), signingCertificatePkcs12.size()));
     if (!bioPkcs12)
         throw OpenSslException("Could not create BIO buffer for PKCS#12 data");