openss: fix various test and Python 3 issues (#47188) (#47235)

(cherry picked from commit 6666b070a97af5735694e2f421c56d12cccbbfd2)

3 files changed, 8 insertions, 6 deletions

diff --git a/changelogs/fragments/openssl-python3.yaml b/changelogs/fragments/openssl-python3.yaml
new file mode 100644
index 0000000000..d6c239e899
--- /dev/null
+++ b/changelogs/fragments/openssl-python3.yaml
@@ -0,0 +1,2 @@
+bugfixes:
+- openssl_csr - fix byte encoding issue on Python 3
diff --git a/lib/ansible/modules/crypto/openssl_csr.py b/lib/ansible/modules/crypto/openssl_csr.py
index c6dfa75fd5..688614f716 100644
--- a/lib/ansible/modules/crypto/openssl_csr.py
+++ b/lib/ansible/modules/crypto/openssl_csr.py
@@ -455,7 +455,7 @@ class CertificateSigningRequest(crypto_utils.OpenSSLObject):
             return _check_keyUsage_(extensions, b'basicConstraints', self.basicConstraints, self.basicConstraints_critical)
 
         def _check_ocspMustStaple(extensions):
-            oms_ext = [ext for ext in extensions if ext.get_short_name() == MUST_STAPLE_NAME and str(ext) == MUST_STAPLE_VALUE]
+            oms_ext = [ext for ext in extensions if to_bytes(ext.get_short_name()) == MUST_STAPLE_NAME and to_bytes(ext) == MUST_STAPLE_VALUE]
             if OpenSSL.SSL.OPENSSL_VERSION_NUMBER < 0x10100000:
                 # Older versions of libssl don't know about OCSP Must Staple
                 oms_ext.extend([ext for ext in extensions if ext.get_short_name() == b'UNDEF' and ext.get_data() == b'\x30\x03\x02\x01\x05'])
diff --git a/test/integration/targets/openssl_privatekey/tests/validate.yml b/test/integration/targets/openssl_privatekey/tests/validate.yml
index 337d839e84..f2e39a37f7 100644
--- a/test/integration/targets/openssl_privatekey/tests/validate.yml
+++ b/test/integration/targets/openssl_privatekey/tests/validate.yml
@@ -1,5 +1,5 @@
 - name: Validate privatekey1 (test - RSA key with size 4096 bits)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey1.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey1.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey1
 
 - name: Validate privatekey1 (assert - RSA key with size 4096 bits)
@@ -9,7 +9,7 @@
 
 
 - name: Validate privatekey2 (test - RSA key with size 2048 bits)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey2.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey2.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey2
 
 - name: Validate privatekey2 (assert - RSA key with size 2048 bits)
@@ -19,7 +19,7 @@
 
 
 - name: Validate privatekey3 (test - DSA key with size 4096 bits)
-  shell: "openssl dsa -noout -text -in {{ output_dir }}/privatekey3.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl dsa -noout -text -in {{ output_dir }}/privatekey3.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey3
 
 - name: Validate privatekey3 (assert - DSA key with size 4096 bits)
@@ -40,7 +40,7 @@
 
 
 - name: Validate privatekey5 (test - Passphrase protected key + idempotence)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey5.pem -passin pass:ansible  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey5.pem -passin pass:ansible  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey5
   # Current version of OS/X that runs in the CI (10.11) does not have an up to date version of the OpenSSL library
   # leading to this test to fail when run in the CI. However, this test has been run for 10.12 and has returned succesfully.
@@ -59,7 +59,7 @@
 
 
 - name: Validate privatekey6 (test - Passphrase protected key with non ascii character)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey6.pem -passin pass:ànsïblé | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey6.pem -passin pass:ànsïblé | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey6
   when: openssl_version.stdout is version('0.9.8zh', '>=')