gost: update gostdsa_vko to follow Nettletmp-update-gostdsa-vko

Update gostdsa_vko() following changes going to be accepted into Nettle.

Signed-off-by: Dmitry Baryshkov <dbaryshkov@gmail.com>

4 files changed, 59 insertions, 29 deletions

diff --git a/configure.ac b/configure.ac
index d566a842b4..7d70d205c7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -663,6 +663,12 @@ AC_CHECK_FUNCS([nettle_chacha_set_counter],
 LIBS=$save_LIBS
 AM_CONDITIONAL(NEED_CHACHA, [test "$have_chacha_set_counter" != "yes"])
 
+# Check for VKO
+save_LIBS=$LIBS
+LIBS="$LIBS $NETTLE_LIBS $HOGWEED_LIBS"
+AC_CHECK_FUNCS(nettle_gostdsa_vko)
+LIBS=$save_LIBS
+
 AC_MSG_CHECKING([whether to build libdane])
 AC_ARG_ENABLE(libdane,
     AS_HELP_STRING([--disable-libdane],
diff --git a/lib/nettle/gost/gostdsa-vko.c b/lib/nettle/gost/gostdsa-vko.c
index 1d55899625..ec6fba5831 100644
--- a/lib/nettle/gost/gostdsa-vko.c
+++ b/lib/nettle/gost/gostdsa-vko.c
@@ -33,6 +33,8 @@
 # include "config.h"
 #endif
 
+#ifndef HAVE_NETTLE_GOSTDSA_VKO
+
 #include <gnutls_int.h>
 
 #include <stdlib.h>
@@ -40,13 +42,24 @@
 #include "ecc/ecc-internal.h"
 #include "gostdsa2.h"
 
-int
-gostdsa_vko(const struct ecc_scalar *key,
+/*
+ * Shared key derivation/key agreement for GOST DSA algorithm.
+ * It is defined in RFC 4357 Section 5.2 and RFC 7836 Section 4.3.1
+ *
+ * Basically shared key is equal to hash(cofactor * ukm * priv * pub). This
+ * function does multiplication. Caller should do hashing on his own.
+ *
+ * UKM is not a secret value (consider it as a nonce).
+ *
+ * For supported GOST curves cofactor is equal to 1.
+ */
+void
+gostdsa_vko(const struct ecc_scalar *priv,
 	    const struct ecc_point *pub,
 	    size_t ukm_length, const uint8_t *ukm,
-	    size_t out_length, uint8_t *out)
+	    uint8_t *out)
 {
-  const struct ecc_curve *ecc = key->ecc;
+  const struct ecc_curve *ecc = priv->ecc;
   unsigned bsize = (ecc_bit_size(ecc) + 7) / 8;
   mp_size_t size = ecc->p.size;
   mp_size_t itch = 4*size + ecc->mul_itch;
@@ -55,24 +68,30 @@ gostdsa_vko(const struct ecc_scalar *key,
   if (itch < 5*size + ecc->h_to_a_itch)
       itch = 5*size + ecc->h_to_a_itch;
 
-  if (pub->ecc != ecc)
-      return 0;
-
-  if (out_length < 2 * bsize) {
-      return 0;
-  }
+  assert (pub->ecc == ecc);
+  assert (priv->ecc == ecc);
+  assert (ukm_length <= bsize);
 
   scratch = gmp_alloc_limbs (itch);
 
-  mpn_set_base256_le (scratch, size, ukm, ukm_length);
-  if (mpn_zero_p (scratch, size))
-    mpn_add_1 (scratch, scratch, size, 1);
-  ecc_mod_mul (&ecc->q, scratch + 3*size, key->p, scratch);
-  ecc->mul (ecc, scratch, scratch + 3*size, pub->p, scratch + 4*size);
-  ecc->h_to_a (ecc, 0, scratch + 3*size, scratch, scratch + 5*size);
-  mpn_get_base256_le (out, bsize, scratch + 3*size, size);
-  mpn_get_base256_le (out+bsize, bsize, scratch + 4*size, size);
-  gmp_free_limbs (scratch, itch);
+#define UKM scratch
+#define TEMP (scratch + 3*size)
+#define XYZ scratch
+#define TEMP_Y (scratch + 4*size)
+
+  mpn_set_base256_le (UKM, size, ukm, ukm_length);
 
-  return 2 * bsize;
+  /* If ukm is 0, set it to 1, otherwise the result will be allways equal to 0,
+   * no matter what private and public keys are. See RFC 4357 referencing GOST
+   * R 34.10-2001 (RFC 5832) Section 6.1 step 2. */
+  if (mpn_zero_p (UKM, size))
+    UKM[0] = 1;
+
+  ecc_mod_mul (&ecc->q, TEMP, priv->p, UKM); /* TEMP = UKM * priv */
+  ecc->mul (ecc, XYZ, TEMP, pub->p, scratch + 4*size); /* XYZ = UKM * priv * pub */
+  ecc->h_to_a (ecc, 0, TEMP, XYZ, scratch + 5*size); /* TEMP = XYZ */
+  mpn_get_base256_le (out, bsize, TEMP, size);
+  mpn_get_base256_le (out+bsize, bsize, TEMP_Y, size);
+  gmp_free_limbs (scratch, itch);
 }
+#endif
diff --git a/lib/nettle/gost/gostdsa2.h b/lib/nettle/gost/gostdsa2.h
index a4b26ab497..61bd31718a 100644
--- a/lib/nettle/gost/gostdsa2.h
+++ b/lib/nettle/gost/gostdsa2.h
@@ -41,19 +41,26 @@
 extern "C" {
 #endif
 
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
 /* Name mangling */
 #define gostdsa_unmask_key _gnutls_gostdsa_unmask_key
-#define gostdsa_vko _gnutls_gostdsa_vko
 
 int
 gostdsa_unmask_key (const struct ecc_curve *ecc,
 		    mpz_t key);
 
-int
-gostdsa_vko(const struct ecc_scalar *key,
+#ifndef HAVE_NETTLE_GOSTDSA_VKO
+
+#define gostdsa_vko _gnutls_gostdsa_vko
+void
+gostdsa_vko(const struct ecc_scalar *priv,
 	    const struct ecc_point *pub,
 	    size_t ukm_length, const uint8_t *ukm,
-	    size_t out_length, uint8_t *out);
+	    uint8_t *out);
+#endif
 
 #ifdef __cplusplus
 }
diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index 1f03bc38a9..15ad4b4e9a 100644
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -486,11 +486,9 @@ dh_cleanup:
 			goto gost_cleanup;
 		}
 
-		out->size = gostdsa_vko(&ecc_priv, &ecc_pub,
-					nonce->size, nonce->data,
-					out->size, out->data);
-		if (out->size == 0)
-			ret = GNUTLS_E_INVALID_REQUEST;
+		gostdsa_vko(&ecc_priv, &ecc_pub,
+			    nonce->size, nonce->data,
+			    out->data);
 
 	      gost_cleanup:
 		ecc_point_clear(&ecc_pub);