Try to deal with the non-standard OpenSSL3 setup in Coin VMs

 * The system's OpenSSL libs might be different (even 1.x based) than
   the 3.x libs Qt is built against.
 * macOS still cannot handle the new default algorithms used by
   OpenSSL 3 for PKCS#12, so we have to create these using "-legacy".
 * Also be lenient in the packager and try to use old certificates, if
   the user specifies them: they might be deployed to a legacy system.

Change-Id: Icbcb8e53d6d0d102bb51b28780712adb4e919401
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Dominik Holland <dominik.holland@qt.io>
(cherry picked from commit b62b9a37b752e153385b3dc9460c4b010b6d3a1a)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

6 files changed, 57 insertions, 17 deletions

diff --git a/src/crypto-lib/libcryptofunction.cpp b/src/crypto-lib/libcryptofunction.cpp
index 142eac25..c485529a 100644
--- a/src/crypto-lib/libcryptofunction.cpp
+++ b/src/crypto-lib/libcryptofunction.cpp
@@ -60,7 +60,7 @@ bool Cryptography::LibCryptoFunctionBase::initialize(bool loadOpenSsl3LegacyProv
         unsigned long version = 0;
 
         if (am_OpenSSL_version_num.functionPointer())
-            version = am_OpenSSL_version_num();  // 1.1
+            version = am_OpenSSL_version_num();  // 1.1 and 3.x
         else if (am_SSLeay.functionPointer())
             version = am_SSLeay(); // 1.0
 
diff --git a/src/tools/appman/appman.cpp b/src/tools/appman/appman.cpp
index 53c518e8..03646bd9 100644
--- a/src/tools/appman/appman.cpp
+++ b/src/tools/appman/appman.cpp
@@ -11,7 +11,6 @@
 #include "logging.h"
 #include "main.h"
 #include "configuration.h"
-#include "packageutilities.h"
 #if !defined(AM_DISABLE_INSTALLER)
 #  include "sudo.h"
 #endif
diff --git a/src/tools/packager/packager.cpp b/src/tools/packager/packager.cpp
index fd0bfffc..babb64dc 100644
--- a/src/tools/packager/packager.cpp
+++ b/src/tools/packager/packager.cpp
@@ -17,6 +17,7 @@
 #include <QtAppManCommon/exception.h>
 #include <QtAppManCommon/qtyaml.h>
 #include <QtAppManCommon/utilities.h>
+#include <QtAppManCrypto/cryptography.h>
 #include <QtAppManPackage/packageutilities.h>
 #include "packagingjob.h"
 
@@ -64,6 +65,9 @@ static Command command(QCommandLineParser &clp)
 
 int main(int argc, char *argv[])
 {
+    // enable OpenSSL3 to load old certificates
+    Cryptography::enableOpenSsl3LegacyProvider();
+
     PackageUtilities::ensureCorrectLocale();
 
     QCoreApplication::setApplicationName(qSL("Qt ApplicationManager Packager"));
diff --git a/tests/data/certificates/create-test-certificates.sh b/tests/data/certificates/create-test-certificates.sh
index 67f1e501..068267b6 100755
--- a/tests/data/certificates/create-test-certificates.sh
+++ b/tests/data/certificates/create-test-certificates.sh
@@ -4,6 +4,12 @@
 # Copyright (C) 2018 Pelagicore AG
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
+#set -x
+set -e
+
+# check basic requirement
+[ ! -e openssl-ca.cnf ] && { echo "Please cd to the tests/data/certificates directory before running this script"; exit 1; }
+
 . ../utilities.sh
 
 rm -f index.txt* serial.txt*
@@ -18,14 +24,49 @@ rm -f other-index.txt* other-serial.txt*
 rm -f other-ca-priv.key other-ca.crt
 rm -f other.csr other.crt other-priv.key other.p12
 
+
+echo "OpenSSL installation check:"
+
+# cater for the most common settings in the CI
+SSL_BIN_PATH=""
+if [ -n "$OPENSSL_DIR" ]; then
+  SSL_BIN_PATH="$OPENSSL_DIR/bin/"
+elif [ -n "$OPENSSL_HOME" ]; then
+  SSL_BIN_PATH="$OPENSSL_HOME/bin/"
+fi
+
+echo " * using openssl at ${SSL_BIN_PATH:-`which openssl`}"
+
+# try to execute and extract the major version number
+SSL_VERSION=$(${SSL_BIN_PATH}openssl version 2>/dev/null || true)
+SSL_MAJOR=$(echo "$SSL_VERSION" | cut -d' ' -f 2 | cut -d'.' -f 1)
+if [ -z "$SSL_VERSION" ] || [ -z "$SSL_MAJOR" ] || ! [ "$SSL_MAJOR" -eq "$SSL_MAJOR" ] 2>/dev/null; then
+  echo -e "$R Failed$W to run or parse the output of$G openssl version$W".
+  exit 1
+fi
+
+echo " * version $SSL_MAJOR (${SSL_VERSION})"
+echo
+
+SSL_PKCS12_EXTRA=''
+if [ "${SSL_MAJOR}" -ge 3 ]; then
+  # if we don't do this, then macOS' SecurityFramework cannot load the PKCS#12 files
+  if [ "$isMac" = "1" ]; then
+    SSL_PKCS12_EXTRA="-legacy"
+    echo " * using -legacy mode for PKCS#12 files for SecurityFramework compatibility"
+  fi
+fi
+
 runSSL()
 {
-  sslOutput=`openssl "$@" 2>&1`
+  set +e
+  sslOutput=`${SSL_BIN_PATH}openssl "$@" 2>&1`
   sslResult=$?
+  set -e
   if [ $sslResult -ne 0 ]; then
-    echo -e "The openssl$R failed with exit code $sslResult$W. The executed command was:"
+    echo -e "Running openssl $R failed with exit code $sslResult$W. The executed command was:"
     echo
-    echo -e "   $G openssl $@$W"
+    echo -e "   $G ${SSL_BIN_PATH}openssl $@$W"
     echo
     echo "The command's output was:"
     echo
@@ -48,7 +89,7 @@ echo '01' > serial.txt
 info "Generating, signing and exporting the store certificate"
 runSSL req -config openssl-store.cnf -newkey rsa:2048 -nodes -keyout store-priv.key -out store.csr
 runSSL ca -batch -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out store.crt -infiles store.csr
-runSSL pkcs12 -export -password pass:password -out store.p12 -inkey store-priv.key -nodes -in store.crt -name "Pelagicore App Store"
+runSSL pkcs12 ${SSL_PKCS12_EXTRA} -export -password pass:password -out store.p12 -inkey store-priv.key -nodes -in store.crt -name "Pelagicore App Store"
 
 info "Generating the developer sub-CA"
 runSSL req -config openssl-devca.cnf -newkey rsa:2048 -nodes -keyout devca-priv.key -out devca.csr
@@ -59,12 +100,12 @@ echo '01' > dev-serial.txt
 info "Generating, signing and exporting the developer certificate #1"
 runSSL req -config openssl-dev1.cnf -newkey rsa:2048 -nodes -keyout dev1-priv.key -out dev1.csr
 runSSL ca -batch -config openssl-devca.cnf -policy signing_policy -extensions signing_req -out dev1.crt -infiles dev1.csr
-runSSL pkcs12 -export -out dev1.p12 -password pass:password -inkey dev1-priv.key -nodes -certfile devca.crt -in dev1.crt -name "Developer 1 Certificate"
+runSSL pkcs12 ${SSL_PKCS12_EXTRA} -export -out dev1.p12 -password pass:password -inkey dev1-priv.key -nodes -certfile devca.crt -in dev1.crt -name "Developer 1 Certificate"
 
 info "Generating, signing and exporting the developer certificate #2"
 runSSL req -config openssl-dev2.cnf -newkey rsa:2048 -nodes -keyout dev2-priv.key -out dev2.csr
 runSSL ca -batch -config openssl-devca.cnf -policy signing_policy -extensions signing_req -out dev2.crt -infiles dev2.csr
-runSSL pkcs12 -export -out dev2.p12 -password pass:password -inkey dev2-priv.key -nodes -certfile devca.crt -in dev2.crt -name "Developer 2 Certificate"
+runSSL pkcs12 ${SSL_PKCS12_EXTRA} -export -out dev2.p12 -password pass:password -inkey dev2-priv.key -nodes -certfile devca.crt -in dev2.crt -name "Developer 2 Certificate"
 
 info "Generating the \"other\" CA"
 # generate self-signed CA cert
@@ -76,7 +117,7 @@ echo '01' > other-serial.txt
 # the double // is needed to get around MSYS hardwired path replacement
 runSSL req -config openssl-other-ca.cnf -batch -subj '//C=DE/ST=Foo/L=Bar/CN=www.other.com' -newkey rsa:2048 -nodes -keyout other-priv.key -out other.csr
 runSSL ca -batch -config openssl-other-ca.cnf -policy signing_policy -extensions signing_req -out other.crt -infiles other.csr
-runSSL pkcs12 -export -out other.p12 -password pass:password -inkey other-priv.key -nodes -certfile other-ca.crt -in other.crt -name "Other Certificate"
+runSSL pkcs12 ${SSL_PKCS12_EXTRA} -export -out other.p12 -password pass:password -inkey other-priv.key -nodes -certfile other-ca.crt -in other.crt -name "Other Certificate"
 
 echo -e "$G All test certificates have been created successfully$W"
 echo
diff --git a/tests/data/create-test-packages.sh b/tests/data/create-test-packages.sh
index 860e0b2d..acfff91a 100755
--- a/tests/data/create-test-packages.sh
+++ b/tests/data/create-test-packages.sh
@@ -7,14 +7,11 @@
 #set -x
 set -e
 
-isWin=0
-isMac=0
-[ "$OS" == "Windows_NT" ] && isWin=1
-[ "$(uname)" == "Darwin" ] && isMac=1
-
 # check basic requirement
 [ ! -d certificates ] && { echo "Please cd to the tests/data directory before running this script"; exit 1; }
 
+. ./utilities.sh
+
 # set a well-known UTF-8 locale: C.UTF-8 is the obvious choice, but macOS doesn't support it
 if [ "$isMac" = "1" ]; then
   export LC_ALL=en_US.UTF-8
@@ -22,8 +19,6 @@ else
   export LC_ALL=C.UTF-8
 fi
 
-. utilities.sh
-
 usage()
 {
      echo "create-test-packages.sh <appman-packager binary>"
diff --git a/tests/data/utilities.sh b/tests/data/utilities.sh
index f90d6bc6..679230ea 100644
--- a/tests/data/utilities.sh
+++ b/tests/data/utilities.sh
@@ -8,7 +8,8 @@ R="\033[0;31m"
 G="\033[0;32m"
 W="\033[0m"
 
-[ `uname` == 'Darwin' ] && isMac=1 || isMac=0
+[ "$OS" == "Windows_NT" ] && isWin=1 || isWin=0
+[ "$(uname)" == "Darwin" ] && isMac=1 || isMac=0
 
 echo()
 {