x509: implement RSA-PSS signature scheme

This patch enables RSA-PSS signature scheme in the X.509 functions and
certtool.

When creating RSA-PSS signature, there are 3 different scenarios:

a. both a private key and a certificate are RSA-PSS
b. the private key is RSA, while the certificate is RSA-PSS
c. both the private key and the certificate are RSA

For (a) and (b), the RSA-PSS parameters are read from the certificate.
Any conflicts in parameters between the private key and the certificate
are reported as an error.

For (c), the sign functions, such as gnutls_x509_crt_privkey_sign() or
gnutls_privkey_sign_data(), shall be instructed to generate an RSA-PSS
signature.  This can be done with the new flag
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

Verification is similar to signing, except for the case (c), use the
flag GNUTLS_VERIFY_USE_RSA_PSS instead of
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

From the command line, certtool has a couple of new options: --rsa-pss
and --rsa-pss-sign.  The --rsa-pss option indicates that the generated
private key or certificate is restricted to RSA-PSS, while the
--rsa-pss-sign option indicates that the generated certificate is signed
with RSA-PSS.

For simplicity, there is no means of choosing arbitrary salt length.
When it is not given by a private key or a certificate, it is
automatically calculated from the underlying hash algorithm and the
RSA modulus bits.

[minor naming changes by nmav]

Signed-off-by: Daiki Ueno <dueno@redhat.com>
Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

1 files changed, 188 insertions, 0 deletions

diff --git a/lib/x509/spki.c b/lib/x509/spki.c
new file mode 100644
index 0000000000..af94ea4a2a
--- /dev/null
+++ b/lib/x509/spki.c
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2017 Red Hat, Inc.
+ *
+ * Authors: Daiki Ueno
+ *
+ * This file is part of GnuTLS.
+ *
+ * The GnuTLS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+#include "gnutls_int.h"
+#include "errors.h"
+#include <common.h>
+#include <x509.h>
+#include <x509_int.h>
+
+/**
+ * gnutls_x509_spki_init:
+ * @spki: A pointer to the type to be initialized
+ *
+ * This function will initialize a SubjectPublicKeyInfo structure used
+ * in PKIX. The structure is used to set additional parameters
+ * in the public key information field of a certificate.
+ *
+ * Returns: On success, %GNUTLS_E_SUCCESS (0) is returned, otherwise a
+ *   negative error value.
+ *
+ * Since: 3.6.0
+ *
+ **/
+int
+gnutls_x509_spki_init(gnutls_x509_spki_t *spki)
+{
+	gnutls_x509_spki_t tmp;
+
+	FAIL_IF_LIB_ERROR;
+
+	tmp =
+	    gnutls_calloc(1, sizeof(gnutls_x509_spki_st));
+
+	if (!tmp)
+		return GNUTLS_E_MEMORY_ERROR;
+
+	*spki = tmp;
+
+	return 0;		/* success */
+}
+
+/**
+ * gnutls_x509_spki_deinit:
+ * @spki: the SubjectPublicKeyInfo structure
+ *
+ * This function will deinitialize a SubjectPublicKeyInfo structure.
+ *
+ * Since: 3.6.0
+ *
+ **/
+void
+gnutls_x509_spki_deinit(gnutls_x509_spki_t spki)
+{
+	gnutls_free(spki);
+}
+
+/**
+ * gnutls_x509_spki_set_pk_algorithm:
+ * @spki: the SubjectPublicKeyInfo structure
+ * @pk: the public key algorithm of type #gnutls_pk_algorithm_t
+ *
+ * This function will set the public key algorithm of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * Since: 3.6.0
+ *
+ **/
+void
+gnutls_x509_spki_set_pk_algorithm(gnutls_x509_spki_t spki,
+				  gnutls_pk_algorithm_t pk)
+{
+	spki->pk = pk;
+}
+
+/**
+ * gnutls_x509_spki_get_pk_algorithm:
+ * @spki: the SubjectPublicKeyInfo structure
+ *
+ * This function will get the public key algorithm of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * Returns: a member of the #gnutls_pk_algorithm_t enumeration on
+ * success, or %GNUTLS_PK_UNKNOWN on error.
+ *
+ * Since: 3.6.0
+ *
+ **/
+int
+gnutls_x509_spki_get_pk_algorithm(gnutls_x509_spki_t spki)
+{
+	return spki->pk;
+}
+
+/**
+ * gnutls_x509_spki_set_digest_algorithm:
+ * @spki: the SubjectPublicKeyInfo structure
+ * @dig: the digest algorithm of type #gnutls_digest_algorithm_t
+ *
+ * This function will set the digest algorithm of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * Since: 3.6.0
+ *
+ **/
+void
+gnutls_x509_spki_set_digest_algorithm(gnutls_x509_spki_t spki,
+				      gnutls_digest_algorithm_t dig)
+{
+	spki->dig = dig;
+}
+
+/**
+ * gnutls_x509_spki_get_digest_algorithm:
+ * @spki: the SubjectPublicKeyInfo structure
+ *
+ * This function will get the digest algorithm of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * Returns: a member of the #gnutls_digest_algorithm_t enumeration on
+ * success, or a %GNUTLS_DIG_UNKNOWN on error.
+ *
+ * Since: 3.6.0
+ *
+ **/
+int
+gnutls_x509_spki_get_digest_algorithm(gnutls_x509_spki_t spki)
+{
+	return spki->dig;
+}
+
+/**
+ * gnutls_x509_spki_set_salt_size:
+ * @spki: the SubjectPublicKeyInfo structure
+ * @salt_size: the size of salt string
+ *
+ * This function will set the salt size parameter of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * The salt is used in the RSA-PSS signature scheme.
+ *
+ * Since: 3.6.0
+ *
+ **/
+void
+gnutls_x509_spki_set_salt_size(gnutls_x509_spki_t spki,
+			       unsigned int salt_size)
+{
+	spki->salt_size = salt_size;
+}
+
+/**
+ * gnutls_x509_spki_get_salt_size:
+ * @spki: the SubjectPublicKeyInfo structure
+ *
+ * This function will get the salt size parameter of a
+ * SubjectPublicKeyInfo structure.
+ *
+ * The salt is used in the RSA-PSS signature scheme.
+ *
+ * Returns: salt size as a positive integer, or zero.
+ *
+ * Since: 3.6.0
+ *
+ **/
+int
+gnutls_x509_spki_get_salt_size(gnutls_x509_spki_t spki)
+{
+	return spki->salt_size;
+}