Handle private keys with lowercase hex digits in DEK-Info

Some tools, for example win-acme, create encrypted private keys in OpenSSL's
traditional format containing lowercase hex digits in the IV part of the
DEK-Info PEM header. These key files are accepted by OpenSSL. Prior to this
patch, GnuTLS did reject these keys with GNUTLS_E_INVALID_REQUEST.

Signed-off-by: Tim Kosse <tim.kosse@filezilla-project.org>
Co-authored-by: Daiki Ueno <ueno@gnu.org>

2 files changed, 41 insertions, 16 deletions

diff --git a/lib/x509/privkey_openssl.c b/lib/x509/privkey_openssl.c
index 9fc70e0322..4ac1ad2672 100644
--- a/lib/x509/privkey_openssl.c
+++ b/lib/x509/privkey_openssl.c
@@ -142,7 +142,7 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
 	gnutls_cipher_hd_t handle;
 	gnutls_cipher_algorithm_t cipher = GNUTLS_CIPHER_UNKNOWN;
 	gnutls_datum_t b64_data;
-	gnutls_datum_t salt, enc_key;
+	gnutls_datum_t salt, enc_key, hex_data;
 	unsigned char *key_data;
 	size_t key_data_size;
 	const char *pem_header = (void *) data->data;
@@ -150,6 +150,7 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
 	ssize_t pem_header_size;
 	int ret;
 	unsigned int i, iv_size, l;
+	size_t salt_size;
 
 	pem_header_size = data->size;
 
@@ -196,27 +197,21 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
 	if (!salt.data)
 		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
 
-	for (i = 0; i < salt.size * 2; i++) {
-		unsigned char x;
-		const char *c = &pem_header[i];
+	hex_data.data = (unsigned char *)pem_header;
+	hex_data.size = salt.size * 2;
+	salt_size = salt.size;
 
-		if (*c >= '0' && *c <= '9')
-			x = (*c) - '0';
-		else if (*c >= 'A' && *c <= 'F')
-			x = (*c) - 'A' + 10;
-		else {
-			gnutls_assert();
+	ret = gnutls_hex_decode(&hex_data, salt.data, &salt_size);
+	if (ret < 0) {
+		gnutls_assert();
+		if (ret == GNUTLS_E_PARSING_ERROR) {
 			/* Invalid salt in encrypted PEM file */
 			ret = GNUTLS_E_INVALID_REQUEST;
-			goto out_salt;
 		}
-		if (i & 1)
-			salt.data[i / 2] |= x;
-		else
-			salt.data[i / 2] = x << 4;
+		goto out_salt;
 	}
 
-	pem_header += salt.size * 2;
+	pem_header += hex_data.size;
 	if (*pem_header != '\r' && *pem_header != '\n') {
 		gnutls_assert();
 		ret = GNUTLS_E_INVALID_REQUEST;
diff --git a/tests/key-openssl.c b/tests/key-openssl.c
index 7800f23be6..f58a8db685 100644
--- a/tests/key-openssl.c
+++ b/tests/key-openssl.c
@@ -95,6 +95,20 @@ const char key2[] =
     "F3bDyqlxSOm7uxF/K3YzI44v8/D8GGnLBTpN+ANBdiY=\n"
     "-----END RSA PRIVATE KEY-----\n";
 
+const char key_lowercase_iv[] =
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "Proc-Type: 4,ENCRYPTED\n"
+    "DEK-Info: AES-256-CBC,c1967f64f92d5c4ef302537b7b50b98f\n"
+    "\n"
+    "iRHj/BbuyHodcEt/cPduzhxOUCwe+o77j7DOepAb7rasr7uRDjFcB2DeB4yvog5q\n"
+    "M46pB5NVqegJCTcFht/90OKXprt2m04ntsCEXXfJ/NQIYP3NsLM+aNiWUL1cxPiZ\n"
+    "6fWp6uaR165F+T5vBRmo6dS3wowHeiHZMiSGuM6CbW+AO5R31og9cUuP2e02GPbq\n"
+    "ZCGyU8RnA6c1caCql/T/5WOIjyaFpJhigBnQc6EoVi3C6XULBQ1Ut9A8gw3gVWda\n"
+    "NBF8sHfwXyKuPhWLZwOM7ZewIOvnesezwW7Tpf2LfMBIe1YQixdsBgM1RvhEN2bl\n"
+    "mYR1L1zfr94z/fWDztq1MYtCBPUJcgrjNLb80xv1qq5hZorTM9gjAeLfT4x9I6/m\n"
+    "57ohSPIR3bXgRZuefjxBhQYthUPcZ+qktrbURcvHNLs=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
 static int good_pwd_cb(void* userdata, int attempt, const char* token_url,
 		       const char* token_label, unsigned int flags,
 		       char* pin, size_t pin_max) {
@@ -199,6 +213,22 @@ void doit(void)
 	}
 	gnutls_x509_privkey_deinit(pkey);
 
+	/* import a key with lowercase hex digits in the iv part of DEK-Info */
+	ret = gnutls_x509_privkey_init(&pkey);
+	if (ret < 0)
+		fail("gnutls_x509_privkey_init: %d\n", ret);
+
+	key.data = (void *) key_lowercase_iv;
+	key.size = sizeof(key_lowercase_iv);
+	ret =
+	    gnutls_x509_privkey_import2(pkey, &key, GNUTLS_X509_FMT_PEM,
+					"123456", 0);
+	if (ret < 0) {
+		fail("gnutls_x509_privkey_import2: %s\n",
+		     gnutls_strerror(ret));
+	}
+	gnutls_x509_privkey_deinit(pkey);
+
 	/*
 	 * Pin callback passwords will only be used if the password supplied to
 	 * gnutls_x509_privkey_import2 in NULL.  Consider possible combinations