experiment: Implement RSA PSS with truncated hash, fix its tests.

Signed-off-by: NIIBE Yutaka <gniibe@fsij.org>

2 files changed, 56 insertions, 29 deletions

diff --git a/cipher/pkey-rsa.c b/cipher/pkey-rsa.c
index 1c9b3c48..44e0cd66 100644
--- a/cipher/pkey-rsa.c
+++ b/cipher/pkey-rsa.c
@@ -42,7 +42,7 @@ _gcry_pkey_rsapss_sign (gcry_pkey_hd_t h,
   const char *md_name;
   gcry_sexp_t s_tmp, s_tmp2;
 
-  if (num_in != 2)
+  if (num_in != 1 && num_in != 2)
     return gpg_error (GPG_ERR_INV_ARG);
 
   if (num_out != 1)
@@ -80,17 +80,27 @@ _gcry_pkey_rsapss_sign (gcry_pkey_hd_t h,
   if (err)
     return err;
 
-  err = sexp_build (&s_msg, NULL,
-                    "(data"
-                    " (flags pss)"
-                    " (hash-algo %s)"
-                    " (value %b)"
-                    " (salt-length %d)"
-                    " (random-override %b))",
-                    md_name,
-                    (int)in_len[0], in[0],
-                    (int)in_len[1],
-                    (int)in_len[1], in[1]);
+  if (num_in == 1)
+    err = sexp_build (&s_msg, NULL,
+		      "(data"
+		      " (flags pss)"
+		      " (hash-algo %s)"
+		      " (value %b)"
+		      " (salt-length %d))",
+		      md_name,
+		      (int)in_len[0], in[0], 0);
+  else
+    err = sexp_build (&s_msg, NULL,
+		      "(data"
+		      " (flags pss)"
+		      " (hash-algo %s)"
+		      " (value %b)"
+		      " (salt-length %d)"
+		      " (random-override %b))",
+		      md_name,
+		      (int)in_len[0], in[0],
+		      (int)in_len[1],
+		      (int)in_len[1], in[1]);
   if (err)
     {
       sexp_release (s_sk);
@@ -143,7 +153,7 @@ _gcry_pkey_rsapss_verify (gcry_pkey_hd_t h,
   gcry_sexp_t s_msg= NULL;
   gcry_sexp_t s_sig= NULL;
 
-  if (num_in != 3)
+  if (num_in != 2 && num_in != 3)
     return gpg_error (GPG_ERR_INV_ARG);
 
   switch (h->rsa.md_algo)
@@ -177,17 +187,27 @@ _gcry_pkey_rsapss_verify (gcry_pkey_hd_t h,
   if (err)
     return err;
 
-  err = sexp_build (&s_msg, NULL,
-                    "(data"
-                    " (flags pss)"
-                    " (hash-algo %s)"
-                    " (value %b)"
-                    " (salt-length %d)"
-                    " (random-override %b))",
-                    md_name,
-                    (int)in_len[0], in[0],
-                    (int)in_len[1],
-                    (int)in_len[1], in[1]);
+  if (num_in == 2)
+    err = sexp_build (&s_msg, NULL,
+		      "(data"
+		      " (flags pss)"
+		      " (hash-algo %s)"
+		      " (value %b)"
+		      " (salt-length %d))",
+		      md_name,
+		      (int)in_len[0], in[0], 0);
+  else
+    err = sexp_build (&s_msg, NULL,
+		      "(data"
+		      " (flags pss)"
+		      " (hash-algo %s)"
+		      " (value %b)"
+		      " (salt-length %d)"
+		      " (random-override %b))",
+		      md_name,
+		      (int)in_len[0], in[0],
+		      (int)in_len[1],
+		      (int)in_len[1], in[1]);
   if (err)
     {
       sexp_release (s_pk);
@@ -196,7 +216,7 @@ _gcry_pkey_rsapss_verify (gcry_pkey_hd_t h,
 
   err = sexp_build (&s_sig, NULL,
                     "(sig-val(rsa(s %b)))",
-                    (int)in_len[2], in[2]);
+                    (int)in_len[num_in-1], in[num_in-1]);
   if (err)
     {
       sexp_release (s_msg);
diff --git a/tests/t-rsa-pss.c b/tests/t-rsa-pss.c
index e72f10e2..2a00a715 100644
--- a/tests/t-rsa-pss.c
+++ b/tests/t-rsa-pss.c
@@ -172,6 +172,7 @@ one_test (const char *n, const char *e, const char *d,
   size_t out_len[1] = { 0 };
   unsigned int flags = 0;
   int md_algo;
+  int num_in;
 
   if (verbose > 1)
     info ("Running test %s\n", sha_alg);
@@ -259,7 +260,13 @@ one_test (const char *n, const char *e, const char *d,
   in[1] = buffer2;
   in_len[1] = buflen2;
 
-  err = gcry_pkey_op (h0, GCRY_PKEY_OP_SIGN, 2, in, in_len, 1, out, out_len);
+  /* SaltVal = 00 means no salt.  */
+  if (buflen2 == 1 && ((char *)buffer2)[0] == 0)
+    num_in = 1;
+  else
+    num_in = 2;
+
+  err = gcry_pkey_op (h0, GCRY_PKEY_OP_SIGN, num_in, in, in_len, 1, out, out_len);
   if (err)
     fail ("gcry_pkey_op failed: %s", gpg_strerror (err));
 
@@ -278,11 +285,11 @@ one_test (const char *n, const char *e, const char *d,
 
   if (!no_verify)
     {
-      in[2] = out[0];
-      in_len[2] = out_len[0];
+      in[num_in] = out[0];
+      in_len[num_in] = out_len[0];
 
       if ((err = gcry_pkey_op (h1, GCRY_PKEY_OP_VERIFY,
-                               3, in, in_len, 0, NULL, 0)))
+                               num_in+1, in, in_len, 0, NULL, 0)))
         fail ("GCRY_PKEY_OP_VERIFY failed for test: %s",
               gpg_strerror (err));
     }