ext/record_size_limit: account for content type octet in TLS 1.3

In TLS 1.3, the protocol maximum of plaintext size is 2^14+1, while
it is 2^14 in TLS 1.2.  To accommodate that, this introduces the
following invariant:
- when the maximum is set by the user with
  gnutls_record_set_max_size(), store it as is.  The value range is
  [511, 16834].
- when the maximum is negotiated through record_size_limit extension,
  it can be [512, 16385].  In TLS 1.3, subtract by 1 to fit in [511,
  16384].

Signed-off-by: Daiki Ueno <dueno@redhat.com>

2 files changed, 34 insertions, 7 deletions

diff --git a/lib/ext/record_size_limit.c b/lib/ext/record_size_limit.c
index def0d57b25..4c7698ea20 100644
--- a/lib/ext/record_size_limit.c
+++ b/lib/ext/record_size_limit.c
@@ -52,6 +52,7 @@ _gnutls_record_size_limit_recv_params(gnutls_session_t session,
 {
 	ssize_t new_size;
 	ssize_t data_size = _data_size;
+	const version_entry_st *vers;
 
 	DECR_LEN(data_size, 2);
 	if (data_size != 0)
@@ -83,8 +84,14 @@ _gnutls_record_size_limit_recv_params(gnutls_session_t session,
 	_gnutls_handshake_log("EXT[%p]: record_size_limit %u negotiated\n",
 			      session, (unsigned)new_size);
 
+	/* subtract 1 octet for content type */
+	vers = get_version(session);
+	if (unlikely(vers == NULL))
+		return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
+
 	session->security_parameters.max_record_recv_size =
-		MIN(new_size, session->security_parameters.max_record_send_size);
+		MIN(new_size - vers->tls13_sem,
+		    session->security_parameters.max_record_send_size);
 
 	return 0;
 }
@@ -96,20 +103,40 @@ _gnutls_record_size_limit_send_params(gnutls_session_t session,
 				      gnutls_buffer_st * extdata)
 {
 	int ret;
+	uint16_t send_size;
 
 	assert(session->security_parameters.max_record_send_size >= 64 &&
 	       session->security_parameters.max_record_send_size <=
 	       DEFAULT_MAX_RECORD_SIZE);
 
+	send_size = session->security_parameters.max_record_send_size;
+
 	if (session->security_parameters.entity == GNUTLS_SERVER) {
+		const version_entry_st *vers;
+
 		/* if we had received the extension and rejected, don't send it */
 		if (session->internals.hsk_flags & HSK_RECORD_SIZE_LIMIT_RECEIVED &&
 		    !(session->internals.hsk_flags & HSK_RECORD_SIZE_LIMIT_NEGOTIATED))
 			return gnutls_assert_val(0);
+
+		/* add 1 octet for content type */
+		vers = get_version(session);
+		if (unlikely(vers == NULL))
+			return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
+
+		send_size += vers->tls13_sem;
+	} else {
+		const version_entry_st *vers;
+
+		/* add 1 octet for content type */
+		vers = _gnutls_version_max(session);
+		if (unlikely(vers == NULL))
+			return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
+
+		send_size += vers->tls13_sem;
 	}
 
-	ret = _gnutls_buffer_append_prefix(extdata, 16,
-					   session->security_parameters.max_record_send_size);
+	ret = _gnutls_buffer_append_prefix(extdata, 16, send_size);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
diff --git a/tests/tls13/prf.c b/tests/tls13/prf.c
index 75daff59d4..fda8ce6843 100644
--- a/tests/tls13/prf.c
+++ b/tests/tls13/prf.c
@@ -130,10 +130,10 @@ static void dump(const char *name, const uint8_t *data, unsigned data_size)
 	} \
 	}
 
-#define KEY_EXP_VALUE "\xfb\xcb\x96\x87\x8c\x64\x8b\x60\xef\xdc\x76\xb0\x7c\x3b\xd1\x50\x1e\xb1\x3f\x39\xb2\x20\x74\x2c\xb2\x76\x12\x9f\xfc\xad\xb9\xce\x1d\x9a"
-#define HELLO_VALUE "\x61\x32\x14\x81\x9b\xa0\x43\xcd\x39\xbf\x63\x18\x7c\xb7\xf3\x02\x65\xab\x2c\xa4\xaf\xbc\x1c\x7a\x1d\xa4\xc5\x28\x8f\x45\x68"
-#define CONTEXT_VALUE "\xa7\x3c\xa7\x59\x94\x33\xb4\x97\x90\x92\x8c\xe2\x39\xda\x56\x42\x4a\xeb\xeb\xab\x73\xc4\x20\xf0\x34\x4f\xda\xf8\x17\xf5\xbd"
-#define NULL_CONTEXT_VALUE "\x66\xa1\x0a\xcb\xfa\x28\x85\x79\xa3\x30\xeb\xc5\xd5\x50\x62\xdd\xb4\x9c\xa7\x0b\x0b\xe0\x28\x03\x18\xfb\x32\x3d\x37\xf2\xe5"
+#define KEY_EXP_VALUE "\xec\x26\x9e\x8c\x5f\xff\x5c\xb2\x60\x4f\x82\xe7\x6b\xb9\x70\x40\xb9\x2d\x2f\xe7\x41\xa8\xe7\xfa\x03\x7c\xe8\x6d\xfa\xda\xc2\xa9\x3f\x58"
+#define HELLO_VALUE "\xd4\x74\x4a\x09\x28\x0a\x99\xb9\xa4\x5b\x51\x5b\x80\xe7\x50\x1c\x16\xca\x57\x78\xf0\xe5\xa1\x94\x6b\x20\x2b\x14\xff\x2b\x53"
+#define CONTEXT_VALUE "\x8d\xde\xea\x58\xab\x90\xaf\x6c\x5c\x7a\x69\xbf\x8a\xd2\x16\xb4\x0f\x75\xb8\x63\xdb\x86\xe7\x66\x04\x59\xac\x57\xe0\x03\x37"
+#define NULL_CONTEXT_VALUE "\x6c\x1a\x10\x1f\xa9\x5a\xfd\xcd\xf4\xcf\x27\x09\x00\xa8\xca\x8e\x8a\x56\xfb\x80\xf0\x0d\xb3\xa6\xe9\x4a\x5f\xe0\x0c\x31\xd9"
 static void check_prfs(gnutls_session_t session)
 {
 	unsigned char key_material[512];