fips: Run digest&sign self tests for RSA and ECC in FIPS mode.

* cipher/ecc.c (selftest_hash_sign): Implement digest & sign KAT
 (selftests_ecdsa): Run the original basic test only with extended tests
 (run_selftests): Pass-through the extended argument
* cipher/rsa.c (selftest_hash_sign_2048): Implement digest & sign KAT
 (selftests_rsa): Run the original basic test only with extended tests
 (run_selftests): Pass-through the extended argument
--

Cherry-picked from master commit:
	1fc7bfc351ba1d7fa31c0c62a24ad78e9e1cfd5b

Signed-off-by: Jakub Jelen <jjelen@redhat.com>

2 files changed, 234 insertions, 12 deletions

diff --git a/cipher/ecc.c b/cipher/ecc.c
index 9f0e7b11..63b0d05e 100644
--- a/cipher/ecc.c
+++ b/cipher/ecc.c
@@ -1679,6 +1679,126 @@ _gcry_pk_ecc_get_sexp (gcry_sexp_t *r_sexp, int mode, mpi_ec_t ec)
  */
 
 static const char *
+selftest_hash_sign (gcry_sexp_t pkey, gcry_sexp_t skey)
+{
+  int md_algo = GCRY_MD_SHA256;
+  gcry_md_hd_t hd = NULL;
+  const char *data_tmpl = "(data (flags rfc6979) (hash %s %b))";
+  /* Sample data from RFC 6979 section A.2.5, hash is of message "sample" */
+  static const char sample_data[] = "sample";
+  static const char sample_data_bad[] = "sbmple";
+  static const char signature_r[] =
+    "efd48b2aacb6a8fd1140dd9cd45e81d69d2c877b56aaf991c34d0ea84eaf3716";
+  static const char signature_s[] =
+    "f7cb1c942d657c41d436c7a1b6e29f65f3e900dbb9aff4064dc4ab2f843acda8";
+
+  const char *errtxt = NULL;
+  gcry_error_t err;
+  gcry_sexp_t sig = NULL;
+  gcry_sexp_t l1 = NULL;
+  gcry_sexp_t l2 = NULL;
+  gcry_mpi_t r = NULL;
+  gcry_mpi_t s = NULL;
+  gcry_mpi_t calculated_r = NULL;
+  gcry_mpi_t calculated_s = NULL;
+  int cmp;
+
+  err = _gcry_md_open (&hd, md_algo, 0);
+  if (err)
+    {
+      errtxt = "gcry_md_open failed";
+      goto leave;
+    }
+
+  _gcry_md_write (hd, sample_data, strlen(sample_data));
+
+  err = _gcry_mpi_scan (&r, GCRYMPI_FMT_HEX, signature_r, 0, NULL);
+  if (!err)
+    err = _gcry_mpi_scan (&s, GCRYMPI_FMT_HEX, signature_s, 0, NULL);
+
+  if (err)
+    {
+      errtxt = "converting data failed";
+      goto leave;
+    }
+
+  err = _gcry_pk_sign_md (&sig, data_tmpl, hd, skey, NULL);
+  if (err)
+    {
+      errtxt = "signing failed";
+      goto leave;
+    }
+
+  /* check against known signature */
+  errtxt = "signature validity failed";
+  l1 = _gcry_sexp_find_token (sig, "sig-val", 0);
+  if (!l1)
+    goto leave;
+  l2 = _gcry_sexp_find_token (l1, "ecdsa", 0);
+  if (!l2)
+    goto leave;
+
+  sexp_release (l1);
+  l1 = l2;
+
+  l2 = _gcry_sexp_find_token (l1, "r", 0);
+  if (!l2)
+    goto leave;
+  calculated_r = _gcry_sexp_nth_mpi (l2, 1, GCRYMPI_FMT_USG);
+  if (!calculated_r)
+    goto leave;
+
+  sexp_release (l2);
+  l2 = _gcry_sexp_find_token (l1, "s", 0);
+  if (!l2)
+    goto leave;
+  calculated_s = _gcry_sexp_nth_mpi (l2, 1, GCRYMPI_FMT_USG);
+  if (!calculated_s)
+    goto leave;
+
+  errtxt = "known sig check failed";
+
+  cmp = _gcry_mpi_cmp (r, calculated_r);
+  if (cmp)
+    goto leave;
+  cmp = _gcry_mpi_cmp (s, calculated_s);
+  if (cmp)
+    goto leave;
+
+  errtxt = NULL;
+
+  /* verify generated signature */
+  err = _gcry_pk_verify_md (sig, data_tmpl, hd, pkey, NULL);
+  if (err)
+    {
+      errtxt = "verify failed";
+      goto leave;
+    }
+
+  _gcry_md_reset(hd);
+  _gcry_md_write (hd, sample_data_bad, strlen(sample_data_bad));
+  err = _gcry_pk_verify_md (sig, data_tmpl, hd, pkey, NULL);
+  if (gcry_err_code (err) != GPG_ERR_BAD_SIGNATURE)
+    {
+      errtxt = "bad signature not detected";
+      goto leave;
+    }
+
+
+ leave:
+  _gcry_md_close (hd);
+  sexp_release (sig);
+  sexp_release (l1);
+  sexp_release (l2);
+  mpi_release (r);
+  mpi_release (s);
+  mpi_release (calculated_r);
+  mpi_release (calculated_s);
+  return errtxt;
+}
+
+
+static const char *
 selftest_sign (gcry_sexp_t pkey, gcry_sexp_t skey)
 {
   /* Sample data from RFC 6979 section A.2.5, hash is of message "sample" */
@@ -1798,7 +1918,7 @@ selftest_sign (gcry_sexp_t pkey, gcry_sexp_t skey)
 
 
 static gpg_err_code_t
-selftests_ecdsa (selftest_report_func_t report)
+selftests_ecdsa (selftest_report_func_t report, int extended)
 {
   const char *what;
   const char *errtxt;
@@ -1826,8 +1946,16 @@ selftests_ecdsa (selftest_report_func_t report)
       goto failed;
     }
 
-  what = "sign";
-  errtxt = selftest_sign (pkey, skey);
+  if (extended)
+    {
+      what = "sign";
+      errtxt = selftest_sign (pkey, skey);
+      if (errtxt)
+        goto failed;
+    }
+
+  what = "digest sign";
+  errtxt = selftest_hash_sign (pkey, skey);
   if (errtxt)
     goto failed;
 
@@ -1848,12 +1976,10 @@ selftests_ecdsa (selftest_report_func_t report)
 static gpg_err_code_t
 run_selftests (int algo, int extended, selftest_report_func_t report)
 {
-  (void)extended;
-
   if (algo != GCRY_PK_ECC)
     return GPG_ERR_PUBKEY_ALGO;
 
-  return selftests_ecdsa (report);
+  return selftests_ecdsa (report, extended);
 }
 
 
diff --git a/cipher/rsa.c b/cipher/rsa.c
index 9f2b36e8..34c8e490 100644
--- a/cipher/rsa.c
+++ b/cipher/rsa.c
@@ -1761,6 +1761,96 @@ compute_keygrip (gcry_md_hd_t md, gcry_sexp_t keyparam)
  */
 
 static const char *
+selftest_hash_sign_2048 (gcry_sexp_t pkey, gcry_sexp_t skey)
+{
+  int md_algo = GCRY_MD_SHA256;
+  gcry_md_hd_t hd = NULL;
+  const char *data_tmpl = "(data (flags pkcs1) (hash %s %b))";
+  static const char sample_data[] =
+    "11223344556677889900aabbccddeeff"
+    "102030405060708090a0b0c0d0f01121";
+  static const char sample_data_bad[] =
+    "11223344556677889900aabbccddeeff"
+    "802030405060708090a0b0c0d0f01121";
+
+  const char *errtxt = NULL;
+  gcry_error_t err;
+  gcry_sexp_t sig = NULL;
+  /* raw signature data reference */
+  const char ref_data[] =
+    "518f41dea3ad884e93eefff8d7ca68a6f4c30d923632e35673651d675cebd652"
+    "a44ed66f6879b18f3d48b2d235b1dd78f6189be1440352cc94231a55c1f93109"
+    "84616b2841c42fe9a6e37be34cd188207209bd028e2fa93e721fbac40c31a068"
+    "1253b312d4e07addb9c7f3d508fa89f218ea7c7f7b9f6a9b1e522c19fa1cd839"
+    "93f9d4ca2f16c3d0b9abafe5e63e848152afc72ce7ee19ea45353116f85209ea"
+    "b9de42129dbccdac8faa461e8e8cc2ae801101cc6add4ba76ccb752030b0e827"
+    "7352b11cdecebae9cdc9a626c4701cd9c85cd287618888c5fae8b4d0ba48915d"
+    "e5cc64e3aee2ba2862d04348ea71f65454f74f9fd1e3108005cc367ca41585a4";
+  gcry_mpi_t ref_mpi = NULL;
+  gcry_mpi_t sig_mpi = NULL;
+
+  err = _gcry_md_open (&hd, md_algo, 0);
+  if (err)
+    {
+      errtxt = "gcry_md_open failed";
+      goto leave;
+    }
+
+  _gcry_md_write (hd, sample_data, sizeof(sample_data));
+
+  err = _gcry_pk_sign_md (&sig, data_tmpl, hd, skey, NULL);
+  if (err)
+    {
+      errtxt = "signing failed";
+      goto leave;
+    }
+
+  err = _gcry_mpi_scan(&ref_mpi, GCRYMPI_FMT_HEX, ref_data, 0, NULL);
+  if (err)
+    {
+      errtxt = "converting ref_data to mpi failed";
+      goto leave;
+    }
+
+  err = _gcry_sexp_extract_param(sig, "sig-val!rsa", "s", &sig_mpi, NULL);
+  if (err)
+    {
+      errtxt = "extracting signature data failed";
+      goto leave;
+    }
+
+  if (mpi_cmp (sig_mpi, ref_mpi))
+    {
+      errtxt = "signature does not match reference data";
+      goto leave;
+    }
+
+  err = _gcry_pk_verify_md (sig, data_tmpl, hd, pkey, NULL);
+  if (err)
+    {
+      errtxt = "verify failed";
+      goto leave;
+    }
+
+  _gcry_md_reset(hd);
+  _gcry_md_write (hd, sample_data_bad, sizeof(sample_data_bad));
+  err = _gcry_pk_verify_md (sig, data_tmpl, hd, pkey, NULL);
+  if (gcry_err_code (err) != GPG_ERR_BAD_SIGNATURE)
+    {
+      errtxt = "bad signature not detected";
+      goto leave;
+    }
+
+
+ leave:
+  sexp_release (sig);
+  _gcry_md_close (hd);
+  _gcry_mpi_release (ref_mpi);
+  _gcry_mpi_release (sig_mpi);
+  return errtxt;
+}
+
+static const char *
 selftest_sign_2048 (gcry_sexp_t pkey, gcry_sexp_t skey)
 {
   static const char sample_data[] =
@@ -1996,7 +2086,7 @@ selftest_encr_2048 (gcry_sexp_t pkey, gcry_sexp_t skey)
 
 
 static gpg_err_code_t
-selftests_rsa (selftest_report_func_t report)
+selftests_rsa (selftest_report_func_t report, int extended)
 {
   const char *what;
   const char *errtxt;
@@ -2024,8 +2114,16 @@ selftests_rsa (selftest_report_func_t report)
       goto failed;
     }
 
-  what = "sign";
-  errtxt = selftest_sign_2048 (pkey, skey);
+  if (extended)
+    {
+      what = "sign";
+      errtxt = selftest_sign_2048 (pkey, skey);
+      if (errtxt)
+        goto failed;
+    }
+
+  what = "digest sign";
+  errtxt = selftest_hash_sign_2048 (pkey, skey);
   if (errtxt)
     goto failed;
 
@@ -2053,12 +2151,10 @@ run_selftests (int algo, int extended, selftest_report_func_t report)
 {
   gpg_err_code_t ec;
 
-  (void)extended;
-
   switch (algo)
     {
     case GCRY_PK_RSA:
-      ec = selftests_rsa (report);
+      ec = selftests_rsa (report, extended);
       break;
     default:
       ec = GPG_ERR_PUBKEY_ALGO;