tls-sig: reverse bytes in TLS signatures for GOST signatures

GOST TLS suites have one peculiarity: CertificateVerify message uses
byte order opposite to the rest of GOST signature usage (BE instead of
LE). So, reverse byte order in signatures in TLS code. For now this
applies only to TLS 1.2 code. GOST TLS 1.3 ciphersuites will also follow
this approach. Legacy TLS 1.0 ciphersuites also had this peculiarity.

Signed-off-by: Dmitry Eremin-Solenikov <dbaryshkov@gmail.com>

3 files changed, 50 insertions, 1 deletions

diff --git a/lib/algorithms.h b/lib/algorithms.h
index a83fec07bb..60556a9a96 100644
--- a/lib/algorithms.h
+++ b/lib/algorithms.h
@@ -338,6 +338,7 @@ int _gnutls_version_mark_disabled(const char *name);
 gnutls_protocol_t _gnutls_protocol_get_id_if_supported(const char *name);
 
 #define GNUTLS_SIGN_FLAG_TLS13_OK	1 /* if it is ok to use under TLS1.3 */
+#define GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE (1 << 1) /* reverse order of bytes in CrtVrfy signature */
 struct gnutls_sign_entry_st {
 	const char *name;
 	const char *oid;
diff --git a/lib/algorithms/sign.c b/lib/algorithms/sign.c
index 6e4393b5dc..167c5fb51b 100644
--- a/lib/algorithms/sign.c
+++ b/lib/algorithms/sign.c
@@ -353,6 +353,7 @@ gnutls_sign_entry_st sign_algorithms[] = {
 	 .id = GNUTLS_SIGN_GOST_512,
 	 .pk = GNUTLS_PK_GOST_12_512,
 	 .hash = GNUTLS_DIG_STREEBOG_512,
+	 .flags = GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE,
 	 .aid = {{8, 65}, SIG_SEM_PRE_TLS12}},
 	/* GOST R 34.10-2012-256 */
 	{.name = "GOSTR341012-256",
@@ -360,6 +361,7 @@ gnutls_sign_entry_st sign_algorithms[] = {
 	 .id = GNUTLS_SIGN_GOST_256,
 	 .pk = GNUTLS_PK_GOST_12_256,
 	 .hash = GNUTLS_DIG_STREEBOG_256,
+	 .flags = GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE,
 	 .aid = {{8, 64}, SIG_SEM_PRE_TLS12}},
 	/* GOST R 34.10-2001 */
 	{.name = "GOSTR341001",
@@ -367,6 +369,7 @@ gnutls_sign_entry_st sign_algorithms[] = {
 	 .id = GNUTLS_SIGN_GOST_94,
 	 .pk = GNUTLS_PK_GOST_01,
 	 .hash = GNUTLS_DIG_GOSTR_94,
+	 .flags = GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE,
 	 .aid = TLS_SIGN_AID_UNKNOWN},
 	/* GOST R 34.10-94 */
 	{.name = "GOSTR341094",
diff --git a/lib/tls-sig.c b/lib/tls-sig.c
index aebd0b1e3f..80514430ab 100644
--- a/lib/tls-sig.c
+++ b/lib/tls-sig.c
@@ -351,6 +351,34 @@ _gnutls_handshake_verify_data(gnutls_session_t session,
 /* Client certificate verify calculations
  */
 
+static void
+_gnutls_reverse_datum(gnutls_datum_t * d)
+{
+	unsigned i;
+
+	for (i = 0; i < d->size / 2; i ++) {
+		uint8_t t = d->data[i];
+		d->data[i] = d->data[d->size - 1 - i];
+		d->data[d->size - 1 - i] = t;
+	}
+}
+
+static int
+_gnutls_create_reverse(const gnutls_datum_t *src, gnutls_datum_t *dst)
+{
+	unsigned int i;
+
+	dst->size = src->size;
+	dst->data = gnutls_malloc(dst->size);
+	if (!dst->data)
+		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
+
+	for (i = 0; i < dst->size; i++)
+		dst->data[i] = src->data[dst->size - 1 - i];
+
+	return 0;
+}
+
 /* this is _gnutls_handshake_verify_crt_vrfy for TLS 1.2
  */
 static int
@@ -363,6 +391,7 @@ _gnutls_handshake_verify_crt_vrfy12(gnutls_session_t session,
 	int ret;
 	gnutls_datum_t dconcat;
 	const gnutls_sign_entry_st *se = _gnutls_sign_to_entry(sign_algo);
+	gnutls_datum_t sig_rev = {NULL, 0};
 
 	ret = _gnutls_session_sign_algo_enabled(session, sign_algo);
 	if (ret < 0)
@@ -374,6 +403,12 @@ _gnutls_handshake_verify_crt_vrfy12(gnutls_session_t session,
 		return gnutls_assert_val(GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER);
 	}
 
+	if (se->flags & GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE) {
+		ret = _gnutls_create_reverse(signature, &sig_rev);
+		if (ret < 0)
+			return gnutls_assert_val(ret);
+	}
+
 	dconcat.data = session->internals.handshake_hash_buffer.data;
 	dconcat.size = session->internals.handshake_hash_buffer_prev_len;
 
@@ -381,7 +416,9 @@ _gnutls_handshake_verify_crt_vrfy12(gnutls_session_t session,
 	 * because we have checked whether the currently used signature
 	 * algorithm is allowed in the session. */
 	ret = gnutls_pubkey_verify_data2(cert->pubkey, sign_algo, verify_flags|GNUTLS_VERIFY_ALLOW_BROKEN,
-					 &dconcat, signature);
+					 &dconcat,
+					 sig_rev.data ? &sig_rev : signature);
+	_gnutls_free_datum(&sig_rev);
 	if (ret < 0)
 		gnutls_assert();
 
@@ -587,6 +624,7 @@ _gnutls_handshake_sign_crt_vrfy12(gnutls_session_t session,
 {
 	gnutls_datum_t dconcat;
 	gnutls_sign_algorithm_t sign_algo;
+	const gnutls_sign_entry_st *se;
 	int ret;
 
 	sign_algo = _gnutls_session_get_sign_algo(session, cert, pkey, 1);
@@ -595,6 +633,10 @@ _gnutls_handshake_sign_crt_vrfy12(gnutls_session_t session,
 		return GNUTLS_E_UNWANTED_ALGORITHM;
 	}
 
+	se = _gnutls_sign_to_entry(sign_algo);
+	if (se == NULL)
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
 	gnutls_sign_algorithm_set_client(session, sign_algo);
 
 	if (unlikely(gnutls_sign_supports_pk_algorithm(sign_algo, pkey->pk_algorithm) == 0))
@@ -613,6 +655,9 @@ _gnutls_handshake_sign_crt_vrfy12(gnutls_session_t session,
 		return ret;
 	}
 
+	if (se->flags & GNUTLS_SIGN_FLAG_CRT_VRFY_REVERSE)
+		_gnutls_reverse_datum(signature);
+
 	return sign_algo;
 }