pkcs: extract PBE-based encryption and decryption functionstmp-cms-support

Extract internal functions used by both PKCS#12 and PKCS#8 to handle
PBE-based encryption of the data.

Signed-off-by: Dmitry Baryshkov <dbaryshkov@gmail.com>

3 files changed, 189 insertions, 264 deletions

diff --git a/lib/x509/pkcs7-crypt.c b/lib/x509/pkcs7-crypt.c
index 109518d191..3a99157ca2 100644
--- a/lib/x509/pkcs7-crypt.c
+++ b/lib/x509/pkcs7-crypt.c
@@ -319,14 +319,9 @@ int
 _gnutls_pkcs7_decrypt_data(const gnutls_datum_t * data,
 			   const char *password, gnutls_datum_t * dec)
 {
-	int result, len;
-	char enc_oid[MAX_OID_SIZE];
+	int result;
 	gnutls_datum_t tmp;
-	asn1_node pasn = NULL, pkcs7_asn = NULL;
-	int params_start, params_end, params_len;
-	struct pbkdf2_params kdf_params;
-	struct pbe_enc_params enc_params;
-	schema_id schema;
+	asn1_node pkcs7_asn = NULL;
 
 	if ((result =
 	     asn1_create_element(_gnutls_get_pkix(),
@@ -344,57 +339,10 @@ _gnutls_pkcs7_decrypt_data(const gnutls_datum_t * data,
 		goto error;
 	}
 
-	/* Check the encryption schema OID
-	 */
-	len = sizeof(enc_oid);
-	result =
-	    asn1_read_value(pkcs7_asn,
+	result = _gnutls_pkcs_pbe_decrypt_data(data, pkcs7_asn, password, &tmp,
 			    "encryptedContentInfo.contentEncryptionAlgorithm.algorithm",
-			    enc_oid, &len);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-
-	if ((result = _gnutls_check_pkcs_cipher_schema(enc_oid)) < 0) {
-		gnutls_assert();
-		goto error;
-	}
-	schema = result;
-
-	/* Get the DER encoding of the parameters.
-	 */
-	result =
-	    asn1_der_decoding_startEnd(pkcs7_asn, data->data, data->size,
 				       "encryptedContentInfo.contentEncryptionAlgorithm.parameters",
-				       &params_start, &params_end);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-	params_len = params_end - params_start + 1;
-
-	result =
-	    _gnutls_read_pkcs_schema_params(&schema, password,
-					    &data->data[params_start],
-					    params_len, &kdf_params,
-					    &enc_params);
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	/* Parameters have been decoded. Now
-	 * decrypt the EncryptedData.
-	 */
-
-	result =
-	    _gnutls_pkcs_raw_decrypt_data(schema, pkcs7_asn,
-					  "encryptedContentInfo.encryptedContent",
-					  password, &kdf_params, &enc_params,
-					  &tmp);
+					  "encryptedContentInfo.encryptedContent");
 	if (result < 0) {
 		gnutls_assert();
 		goto error;
@@ -407,7 +355,6 @@ _gnutls_pkcs7_decrypt_data(const gnutls_datum_t * data,
 	return 0;
 
  error:
-	asn1_delete_structure(&pasn);
 	asn1_delete_structure2(&pkcs7_asn, ASN1_DELETE_FLAG_ZEROIZE);
 	return result;
 }
@@ -419,7 +366,7 @@ _gnutls_pkcs7_data_enc_info(const gnutls_datum_t * data,
 {
 	int result, len;
 	char enc_oid[MAX_OID_SIZE];
-	asn1_node pasn = NULL, pkcs7_asn = NULL;
+	asn1_node pkcs7_asn = NULL;
 	int params_start, params_end, params_len;
 	struct pbe_enc_params enc_params;
 	schema_id schema;
@@ -498,7 +445,6 @@ _gnutls_pkcs7_data_enc_info(const gnutls_datum_t * data,
 	return 0;
 
  error:
-	asn1_delete_structure(&pasn);
 	asn1_delete_structure2(&pkcs7_asn, ASN1_DELETE_FLAG_ZEROIZE);
 	return result;
 }
@@ -512,17 +458,7 @@ _gnutls_pkcs7_encrypt_data(schema_id schema,
 			   const char *password, gnutls_datum_t * enc)
 {
 	int result;
-	gnutls_datum_t key = { NULL, 0 };
-	gnutls_datum_t tmp = { NULL, 0 };
 	asn1_node pkcs7_asn = NULL;
-	struct pbkdf2_params kdf_params;
-	struct pbe_enc_params enc_params;
-	const struct pkcs_cipher_schema_st *s;
-
-	s = _gnutls_pkcs_schema_get(schema);
-	if (s == NULL || s->decrypt_only) {
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-	}
 
 	if ((result =
 	     asn1_create_element(_gnutls_get_pkix(),
@@ -533,60 +469,15 @@ _gnutls_pkcs7_encrypt_data(schema_id schema,
 		goto error;
 	}
 
-	result =
-	    asn1_write_value(pkcs7_asn,
+	result = _gnutls_pkcs_pbe_encrypt_data(pkcs7_asn, schema, data, password,
 			     "encryptedContentInfo.contentEncryptionAlgorithm.algorithm",
-			     s->write_oid, 1);
-
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-
-	/* Generate a symmetric key.
-	 */
-
-	result =
-	    _gnutls_pkcs_generate_key(schema, password, &kdf_params,
-				      &enc_params, &key);
+			     "encryptedContentInfo.contentEncryptionAlgorithm.parameters",
+			     "encryptedContentInfo.encryptedContent");
 	if (result < 0) {
 		gnutls_assert();
 		goto error;
 	}
 
-	result = _gnutls_pkcs_write_schema_params(schema, pkcs7_asn,
-						  "encryptedContentInfo.contentEncryptionAlgorithm.parameters",
-						  &kdf_params, &enc_params);
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	/* Parameters have been encoded. Now
-	 * encrypt the Data.
-	 */
-	result = _gnutls_pkcs_raw_encrypt_data(data, &enc_params, &key, &tmp);
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	/* write the encrypted data.
-	 */
-	result =
-	    asn1_write_value(pkcs7_asn,
-			     "encryptedContentInfo.encryptedContent",
-			     tmp.data, tmp.size);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-
-	_gnutls_free_datum(&tmp);
-	_gnutls_free_key_datum(&key);
-
 	/* Now write the rest of the pkcs-7 stuff.
 	 */
 
@@ -624,8 +515,6 @@ _gnutls_pkcs7_encrypt_data(schema_id schema,
 	}
 
  error:
-	_gnutls_free_key_datum(&key);
-	_gnutls_free_datum(&tmp);
 	asn1_delete_structure2(&pkcs7_asn, ASN1_DELETE_FLAG_ZEROIZE);
 	return result;
 }
@@ -1115,7 +1004,7 @@ _gnutls_pbes2_string_to_key(unsigned int pass_len, const char *password,
 			     kdf_params->iter_count, key, key_size);
 }
 
-int
+static int
 _gnutls_pkcs_raw_decrypt_data(schema_id schema, asn1_node pkcs8_asn,
 			      const char *root, const char *_password,
 			      const struct pbkdf2_params *kdf_params,
@@ -1710,27 +1599,6 @@ _gnutls_pkcs_write_schema_params(schema_id schema, asn1_node pkcs8_asn,
 
 }
 
-int
-_gnutls_pkcs_raw_encrypt_data(const gnutls_datum_t * plain,
-			      const struct pbe_enc_params *enc_params,
-			      const gnutls_datum_t * key, gnutls_datum_t * encrypted)
-{
-	int result;
-	gnutls_datum_t d_iv;
-	const cipher_entry_st *ce;
-
-	ce = cipher_to_entry(enc_params->cipher);
-
-	d_iv.data = (uint8_t *) enc_params->iv;
-	d_iv.size = enc_params->iv_size;
-
-	result = _gnutls_pkcs7_encrypt_int(ce, key, &d_iv, plain, encrypted);
-	if (result < 0)
-		return gnutls_assert_val(result);
-
-	return 0;
-}
-
 int _gnutls_pkcs7_encrypt_int(const cipher_entry_st *ce, const gnutls_datum_t *key, const gnutls_datum_t *iv, const gnutls_datum_t *plain, gnutls_datum_t *enc)
 {
 	int ret;
@@ -1779,3 +1647,163 @@ int _gnutls_pkcs7_encrypt_int(const cipher_entry_st *ce, const gnutls_datum_t *k
 
 	return 0;
 }
+
+int
+_gnutls_pkcs_pbe_encrypt_data(asn1_node asn,
+			      schema_id schema,
+			      const gnutls_datum_t * plain,
+			      const char *password,
+			      const char *algo,
+			      const char *params,
+			      const char *content)
+{
+	int result;
+	gnutls_datum_t key = { NULL, 0 };
+	gnutls_datum_t tmp = { NULL, 0 };
+	struct pbkdf2_params kdf_params;
+	struct pbe_enc_params enc_params;
+	const struct pkcs_cipher_schema_st *s;
+	gnutls_datum_t d_iv;
+	const cipher_entry_st *ce;
+
+	s = _gnutls_pkcs_schema_get(schema);
+	if (s == NULL || s->decrypt_only) {
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+	}
+
+	/* Write the encryption schema OID
+	 */
+	result = asn1_write_value(asn, algo, s->write_oid, 1);
+	if (result != ASN1_SUCCESS) {
+		gnutls_assert();
+		result = _gnutls_asn2err(result);
+		goto error;
+	}
+
+	/* Generate a symmetric key.
+	 */
+
+	result =
+	    _gnutls_pkcs_generate_key(schema, password, &kdf_params, &enc_params, &key);
+	if (result < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	result =
+	    _gnutls_pkcs_write_schema_params(schema, asn, params,
+				&kdf_params, &enc_params);
+	if (result < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	/* Parameters have been encoded. Now
+	 * encrypt the Data.
+	 */
+	ce = cipher_to_entry(enc_params.cipher);
+
+	d_iv.data = (uint8_t *) enc_params.iv;
+	d_iv.size = enc_params.iv_size;
+
+	result = _gnutls_pkcs7_encrypt_int(ce, &key, &d_iv, plain, &tmp);
+	if (result < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	/* write the encrypted data.
+	 */
+	result =
+	    asn1_write_value(asn, content, tmp.data, tmp.size);
+	if (result != ASN1_SUCCESS) {
+		gnutls_assert();
+		result = _gnutls_asn2err(result);
+		goto error;
+	}
+
+	_gnutls_free_datum(&tmp);
+	_gnutls_free_key_datum(&key);
+
+error:
+	_gnutls_free_key_datum(&key);
+	_gnutls_free_datum(&tmp);
+	return result;
+
+}
+
+int
+_gnutls_pkcs_pbe_decrypt_data(const gnutls_datum_t * data, asn1_node asn,
+			   const char *password, gnutls_datum_t * dec,
+			   const char *algo,
+			   const char *params,
+			   const char *content)
+{
+	int result, len;
+	char enc_oid[MAX_OID_SIZE];
+	gnutls_datum_t tmp;
+	int params_start, params_end, params_len;
+	struct pbkdf2_params kdf_params;
+	struct pbe_enc_params enc_params;
+	schema_id schema;
+
+	/* Check the encryption schema OID
+	 */
+	len = sizeof(enc_oid);
+	result =
+	    asn1_read_value(asn, algo, enc_oid, &len);
+	if (result != ASN1_SUCCESS) {
+		gnutls_assert();
+		result = _gnutls_asn2err(result);
+		goto error;
+	}
+
+	if ((result = _gnutls_check_pkcs_cipher_schema(enc_oid)) < 0) {
+		gnutls_assert();
+		goto error;
+	}
+	schema = result;
+
+	/* Get the DER encoding of the parameters.
+	 */
+	result =
+	    asn1_der_decoding_startEnd(asn, data->data, data->size,
+				       params,
+				       &params_start, &params_end);
+	if (result != ASN1_SUCCESS) {
+		gnutls_assert();
+		result = _gnutls_asn2err(result);
+		goto error;
+	}
+	params_len = params_end - params_start + 1;
+
+	result =
+	    _gnutls_read_pkcs_schema_params(&schema, password,
+					    &data->data[params_start],
+					    params_len, &kdf_params,
+					    &enc_params);
+	if (result < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	/* Parameters have been decoded. Now
+	 * decrypt the EncryptedData.
+	 */
+
+	result =
+	    _gnutls_pkcs_raw_decrypt_data(schema, asn, content,
+					  password, &kdf_params, &enc_params,
+					  &tmp);
+	if (result < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	*dec = tmp;
+
+	return 0;
+
+ error:
+	return result;
+}
diff --git a/lib/x509/pkcs7_int.h b/lib/x509/pkcs7_int.h
index c0eec11338..59142c3c5c 100644
--- a/lib/x509/pkcs7_int.h
+++ b/lib/x509/pkcs7_int.h
@@ -84,16 +84,19 @@ _gnutls_decrypt_pbes1_des_md5_data(const char *password,
 int _gnutls_check_pkcs_cipher_schema(const char *oid);
 
 int
-_gnutls_pkcs_raw_decrypt_data(schema_id schema, asn1_node pkcs8_asn,
-	     const char *root, const char *password,
-	     const struct pbkdf2_params *kdf_params,
-	     const struct pbe_enc_params *enc_params,
-	     gnutls_datum_t *decrypted_data);
-
+_gnutls_pkcs_pbe_encrypt_data(asn1_node asn,
+			      schema_id schema,
+			      const gnutls_datum_t * plain,
+			      const char *password,
+			      const char *algo,
+			      const char *params,
+			      const char *content);
 int
-_gnutls_pkcs_raw_encrypt_data(const gnutls_datum_t * plain,
-	     const struct pbe_enc_params *enc_params,
-	     const gnutls_datum_t * key, gnutls_datum_t * encrypted);
+_gnutls_pkcs_pbe_decrypt_data(const gnutls_datum_t * data, asn1_node asn,
+			   const char *password, gnutls_datum_t * dec,
+			   const char *algo,
+			   const char *params,
+			   const char *content);
 
 int _gnutls_pkcs7_encrypt_int(const cipher_entry_st *ce, const gnutls_datum_t *key, const gnutls_datum_t *iv, const gnutls_datum_t *plain, gnutls_datum_t *enc);
 int _gnutls_pkcs7_decrypt_int(const cipher_entry_st *ce, const gnutls_datum_t *key, const gnutls_datum_t *iv, gnutls_datum_t *data);
diff --git a/lib/x509/privkey_pkcs8.c b/lib/x509/privkey_pkcs8.c
index c22ece9456..b1f0e1f888 100644
--- a/lib/x509/privkey_pkcs8.c
+++ b/lib/x509/privkey_pkcs8.c
@@ -318,17 +318,7 @@ encode_to_pkcs8_key(schema_id schema, const gnutls_datum_t * der_key,
 		    const char *password, asn1_node * out)
 {
 	int result;
-	gnutls_datum_t key = { NULL, 0 };
-	gnutls_datum_t tmp = { NULL, 0 };
 	asn1_node pkcs8_asn = NULL;
-	struct pbkdf2_params kdf_params;
-	struct pbe_enc_params enc_params;
-	const struct pkcs_cipher_schema_st *s;
-
-	s = _gnutls_pkcs_schema_get(schema);
-	if (s == NULL || s->decrypt_only) {
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-	}
 
 	if ((result =
 	     asn1_create_element(_gnutls_get_pkix(),
@@ -338,67 +328,20 @@ encode_to_pkcs8_key(schema_id schema, const gnutls_datum_t * der_key,
 		return _gnutls_asn2err(result);
 	}
 
-	/* Write the encryption schema OID
-	 */
-	result =
-	    asn1_write_value(pkcs8_asn, "encryptionAlgorithm.algorithm",
-			     s->write_oid, 1);
-
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-
-	/* Generate a symmetric key.
-	 */
-
-	result =
-	    _gnutls_pkcs_generate_key(schema, password, &kdf_params, &enc_params, &key);
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	result =
-	    _gnutls_pkcs_write_schema_params(schema, pkcs8_asn,
-				"encryptionAlgorithm.parameters",
-				&kdf_params, &enc_params);
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	/* Parameters have been encoded. Now
-	 * encrypt the Data.
-	 */
-	result = _gnutls_pkcs_raw_encrypt_data(der_key, &enc_params, &key, &tmp);
+	result = _gnutls_pkcs_pbe_encrypt_data(pkcs8_asn, schema, der_key, password,
+			"encryptionAlgorithm.algorithm",
+			"encryptionAlgorithm.parameters",
+			"encryptedData");
 	if (result < 0) {
 		gnutls_assert();
 		goto error;
 	}
 
-	/* write the encrypted data.
-	 */
-	result =
-	    asn1_write_value(pkcs8_asn, "encryptedData", tmp.data,
-			     tmp.size);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-
-	_gnutls_free_datum(&tmp);
-	_gnutls_free_key_datum(&key);
-
 	*out = pkcs8_asn;
 
 	return 0;
 
       error:
-	_gnutls_free_key_datum(&key);
-	_gnutls_free_datum(&tmp);
 	asn1_delete_structure2(&pkcs8_asn, ASN1_DELETE_FLAG_ZEROIZE);
 	return result;
 }
@@ -720,62 +663,13 @@ static int pkcs8_key_decrypt(const gnutls_datum_t * raw_key,
 			     asn1_node pkcs8_asn, const char *password,
 			     gnutls_x509_privkey_t pkey)
 {
-	int result, len;
-	char enc_oid[MAX_OID_SIZE];
+	int result;
 	gnutls_datum_t tmp = {NULL, 0};
-	int params_start, params_end, params_len;
-	struct pbkdf2_params kdf_params;
-	struct pbe_enc_params enc_params;
-	schema_id schema;
 
-	/* Check the encryption schema OID
-	 */
-	len = sizeof(enc_oid);
-	result =
-	    asn1_read_value(pkcs8_asn, "encryptionAlgorithm.algorithm",
-			    enc_oid, &len);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		goto error;
-	}
-
-	if ((result = _gnutls_check_pkcs_cipher_schema(enc_oid)) < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	schema = result;
-
-	/* Get the DER encoding of the parameters.
-	 */
-	result =
-	    asn1_der_decoding_startEnd(pkcs8_asn, raw_key->data,
-				       raw_key->size,
+	result = _gnutls_pkcs_pbe_decrypt_data(raw_key, pkcs8_asn, password, &tmp,
+				       "encryptionAlgorithm.algorithm",
 				       "encryptionAlgorithm.parameters",
-				       &params_start, &params_end);
-	if (result != ASN1_SUCCESS) {
-		gnutls_assert();
-		result = _gnutls_asn2err(result);
-		goto error;
-	}
-	params_len = params_end - params_start + 1;
-
-	result =
-	    _gnutls_read_pkcs_schema_params(&schema, password,
-				    &raw_key->data[params_start],
-				    params_len, &kdf_params, &enc_params);
-
-	if (result < 0) {
-		gnutls_assert();
-		goto error;
-	}
-
-	/* Parameters have been decoded. Now
-	 * decrypt the EncryptedData.
-	 */
-	result =
-	    _gnutls_pkcs_raw_decrypt_data(schema, pkcs8_asn, "encryptedData", password,
-			 &kdf_params, &enc_params, &tmp);
+				       "encryptedData");
 	if (result < 0) {
 		gnutls_assert();
 		result = GNUTLS_E_DECRYPTION_FAILED;