record: adjusted overhead calculation for TLS1.3

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

3 files changed, 26 insertions, 14 deletions

diff --git a/lib/constate.c b/lib/constate.c
index 8bf4cd1224..45a4bdc1df 100644
--- a/lib/constate.c
+++ b/lib/constate.c
@@ -476,10 +476,15 @@ int _gnutls_epoch_set_keys(gnutls_session_t session, uint16_t epoch, hs_stage_t
 			return gnutls_assert_val(ret);
 	}
 
-	session->internals.max_recv_size = _gnutls_record_overhead(params->cipher, params->mac, params->etm, 0);
+	if (ver->tls13_sem) {
+		session->internals.max_recv_size = 256;
+	} else {
+		session->internals.max_recv_size = _gnutls_record_overhead(ver, params->cipher, params->mac, params->etm, 0);
+		if (session->internals.allow_large_records != 0)
+			session->internals.max_recv_size += EXTRA_COMP_SIZE;
+	}
+
 	session->internals.max_recv_size += session->security_parameters.max_record_recv_size + RECORD_HEADER_SIZE(session);
-	if (session->internals.allow_large_records != 0)
-		session->internals.max_recv_size += EXTRA_COMP_SIZE;
 
 	_dtls_reset_window(params);
 
diff --git a/lib/dtls.c b/lib/dtls.c
index de1b090cbf..5827eac978 100644
--- a/lib/dtls.c
+++ b/lib/dtls.c
@@ -493,10 +493,11 @@ void gnutls_dtls_set_mtu(gnutls_session_t session, unsigned int mtu)
 	session->internals.dtls.mtu = MIN(mtu, DEFAULT_MAX_RECORD_SIZE);
 }
 
-int _gnutls_record_overhead(const cipher_entry_st * cipher,
-			   const mac_entry_st * mac,
-			   unsigned etm,
-			   unsigned est_data)
+int _gnutls_record_overhead(const version_entry_st *ver,
+			    const cipher_entry_st * cipher,
+			    const mac_entry_st * mac,
+			    unsigned etm,
+			    unsigned est_data)
 {
 	int total = 0;
 	int ret, blocksize;
@@ -506,7 +507,8 @@ int _gnutls_record_overhead(const cipher_entry_st * cipher,
 		return 0;
 
 	if (mac->id == GNUTLS_MAC_AEAD) {
-		total += cipher->explicit_iv;
+		if (!ver->tls13_sem)
+			total += cipher->explicit_iv;
 		total += _gnutls_cipher_get_tag_size(cipher);
 	} else {
 		ret = _gnutls_mac_get_algo_len(mac);
@@ -591,7 +593,7 @@ size_t gnutls_est_record_overhead_size(gnutls_protocol_t version,
 	else
 		total = DTLS_RECORD_HEADER_SIZE;
 
-	total += _gnutls_record_overhead(c, m, 0, 0);
+	total += _gnutls_record_overhead(v, c, m, 0, 0);
 
 	return total;
 }
@@ -612,13 +614,14 @@ static int record_overhead_rt(gnutls_session_t session, unsigned est_data)
 
 	if (session->internals.initial_negotiation_completed == 0)
 		return GNUTLS_E_INVALID_REQUEST;
-
 	ret = _gnutls_epoch_get(session, EPOCH_WRITE_CURRENT, &params);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
-	return _gnutls_record_overhead(params->cipher, params->mac,
-			       params->etm, est_data);
+	ret = _gnutls_record_overhead(get_version(session), params->cipher,
+				       params->mac,
+				       params->etm, est_data);
+	return ret;
 }
 
 /**
@@ -635,6 +638,7 @@ static int record_overhead_rt(gnutls_session_t session, unsigned est_data)
 size_t gnutls_record_overhead_size(gnutls_session_t session)
 {
 	const version_entry_st *v = get_version(session);
+	int ret;
 	size_t total;
 
 	if (v->transport == GNUTLS_STREAM)
@@ -642,7 +646,9 @@ size_t gnutls_record_overhead_size(gnutls_session_t session)
 	else
 		total = DTLS_RECORD_HEADER_SIZE;
 
-	total += record_overhead_rt(session, 0);
+	ret = record_overhead_rt(session, 0);
+	if (ret >= 0)
+		total += ret;
 
 	return total;
 }
diff --git a/lib/dtls.h b/lib/dtls.h
index c99fdca91c..901e567e49 100644
--- a/lib/dtls.h
+++ b/lib/dtls.h
@@ -111,7 +111,8 @@ inline static void _dtls_async_timer_check(gnutls_session_t session)
 	}
 }
 
-int _gnutls_record_overhead(const cipher_entry_st * cipher,
+int _gnutls_record_overhead(const version_entry_st *ver,
+			    const cipher_entry_st * cipher,
 			    const mac_entry_st * mac,
 			    unsigned etm,
 			    unsigned est_data);