h2645parser: Catch overflows in AVC/HEVC NAL unit length calculations

Offset and size are stored as 32 bit guint and might overflow when
adding the nal_length_size, so let's avoid that.

For the size this would happen if the AVC/HEVC NAL unit size happens to
be stored in 4 bytes and is 4294967292 or higher, which is likely
corrupted data anyway.

For the offset this is something for the caller of these functions to
take care of but is unlikely to happen as it would require parsing on a
>4GB buffer.

Allowing these overflows causes all kinds of follow-up bugs in the
h2645parse elements, ranging from infinite loops and memory leaks to
potential memory corruptions.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2103>

2 files changed, 30 insertions, 2 deletions

diff --git a/gst-libs/gst/codecparsers/gsth264parser.c b/gst-libs/gst/codecparsers/gsth264parser.c
index 012f1d0d7..68aa25068 100644
--- a/gst-libs/gst/codecparsers/gsth264parser.c
+++ b/gst-libs/gst/codecparsers/gsth264parser.c
@@ -1556,6 +1556,14 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1570,7 +1578,13 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H264_PARSER_NO_NAL_END;
diff --git a/gst-libs/gst/codecparsers/gsth265parser.c b/gst-libs/gst/codecparsers/gsth265parser.c
index 6c1759331..99cb23228 100644
--- a/gst-libs/gst/codecparsers/gsth265parser.c
+++ b/gst-libs/gst/codecparsers/gsth265parser.c
@@ -1531,6 +1531,14 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1545,7 +1553,13 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H265_PARSER_NO_NAL_END;