rsa: Prevent usage of long salt in FIPS mode

* cipher/rsa-common.c (_gcry_rsa_pss_encode): Prevent usage of large
  salt lengths
  (_gcry_rsa_pss_verify): Ditto.
* tests/basic.c (check_pubkey_sign): Check longer salt length fails in
  FIPS mode
* tests/t-rsa-pss.c (one_test_sexp): Fix function name in error message

--

Backport the master commit of:
	bf1e62e59200b2046680d1d3d1599facc88cfe63

3 files changed, 33 insertions, 2 deletions

diff --git a/cipher/rsa-common.c b/cipher/rsa-common.c
index 233ddb2d..61cd60a4 100644
--- a/cipher/rsa-common.c
+++ b/cipher/rsa-common.c
@@ -809,6 +809,13 @@ _gcry_rsa_pss_encode (gcry_mpi_t *r_result, unsigned int nbits, int algo,
   hlen = _gcry_md_get_algo_dlen (algo);
   gcry_assert (hlen);  /* We expect a valid ALGO here.  */
 
+  /* The FIPS 186-4 Section 5.5 allows only 0 <= sLen <= hLen */
+  if (fips_mode () && saltlen > hlen)
+    {
+      rc = GPG_ERR_INV_ARG;
+      goto leave;
+    }
+
   /* Allocate a help buffer and setup some pointers.  */
   buflen = 8 + hlen + saltlen + (emlen - hlen - 1);
   buf = xtrymalloc (buflen);
@@ -950,6 +957,13 @@ _gcry_rsa_pss_verify (gcry_mpi_t value, int hashed_already,
   hlen = _gcry_md_get_algo_dlen (algo);
   gcry_assert (hlen);  /* We expect a valid ALGO here.  */
 
+  /* The FIPS 186-4 Section 5.5 allows only 0 <= sLen <= hLen */
+  if (fips_mode () && saltlen > hlen)
+    {
+      rc = GPG_ERR_INV_ARG;
+      goto leave;
+    }
+
   /* Allocate a help buffer and setup some pointers.
      This buffer is used for two purposes:
         +------------------------------+-------+
diff --git a/tests/basic.c b/tests/basic.c
index 99a50175..94a07320 100644
--- a/tests/basic.c
+++ b/tests/basic.c
@@ -15303,6 +15303,7 @@ check_pubkey_sign (int n, gcry_sexp_t skey, gcry_sexp_t pkey, int algo,
     const char *data;
     int algo;
     int expected_rc;
+    int flags;
   } datas[] =
     {
       { "(data\n (flags pkcs1)\n"
@@ -15373,6 +15374,22 @@ check_pubkey_sign (int n, gcry_sexp_t skey, gcry_sexp_t pkey, int algo,
         " (random-override #4253647587980912233445566778899019283747#))\n",
 	GCRY_PK_RSA,
 	0 },
+      { "(data\n (flags pss)\n"
+	" (hash-algo sha256)\n"
+	" (value #11223344556677889900AABBCCDDEEFF#)\n"
+	" (salt-length 2:32)\n"
+        " (random-override #42536475879809122334455667788990192837465564738291"
+                           "00122334455667#))\n",
+	GCRY_PK_RSA,
+	0 },
+      { "(data\n (flags pss)\n"
+	" (hash-algo sha256)\n"
+	" (value #11223344556677889900AABBCCDDEEFF#)\n"
+	" (salt-length 2:33)\n"
+        " (random-override #42536475879809122334455667788990192837465564738291"
+                           "0012233445566778#))\n",
+	GCRY_PK_RSA,
+	0, FLAG_NOFIPS },
       { NULL }
     };
 
@@ -15396,7 +15413,7 @@ check_pubkey_sign (int n, gcry_sexp_t skey, gcry_sexp_t pkey, int algo,
 	die ("converting data failed: %s\n", gpg_strerror (rc));
 
       rc = gcry_pk_sign (&sig, hash, skey);
-      if (in_fips_mode && (flags & FLAG_NOFIPS))
+      if (in_fips_mode && (flags & FLAG_NOFIPS || datas[dataidx].flags & FLAG_NOFIPS))
         {
           if (!rc)
             fail ("gcry_pk_sign did not fail as expected in FIPS mode\n");
diff --git a/tests/t-rsa-pss.c b/tests/t-rsa-pss.c
index c5f90116..82dd54b3 100644
--- a/tests/t-rsa-pss.c
+++ b/tests/t-rsa-pss.c
@@ -340,7 +340,7 @@ one_test_sexp (const char *n, const char *e, const char *d,
     snprintf (p, 3, "%02x", out[i]);
   if (strcmp (sig_string, s))
     {
-      fail ("gcry_pkhash_sign failed: %s",
+      fail ("gcry_pk_hash_sign failed: %s",
             "wrong value returned");
       info ("  expected: '%s'", s);
       info ("       got: '%s'", sig_string);