lavc: check for overflow in init_get_bits

Fix an undefined behaviour and make the function return a proper
error in case of overflow.

CC: libav-stable@libav.org

1 files changed, 15 insertions, 7 deletions

diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h
index c56a2c2d10..16cfd5e0fd 100644
--- a/libavcodec/get_bits.h
+++ b/libavcodec/get_bits.h
@@ -362,20 +362,27 @@ static inline int check_marker(GetBitContext *s, const char *msg)
 }
 
 /**
- * Inititalize GetBitContext.
- * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes larger than the actual read bits
- * because some optimized bitstream readers read 32 or 64 bit at once and could read over the end
+ * Initialize GetBitContext.
+ * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes
+ *        larger than the actual read bits because some optimized bitstream
+ *        readers read 32 or 64 bit at once and could read over the end
  * @param bit_size the size of the buffer in bits
+ * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow.
  */
-static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer,
-                                 int bit_size)
+static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer,
+                                int bit_size)
 {
-    int buffer_size = (bit_size+7)>>3;
-    if (buffer_size < 0 || bit_size < 0) {
+    int buffer_size;
+    int ret = 0;
+
+    if (bit_size > INT_MAX - 7 || bit_size <= 0) {
         buffer_size = bit_size = 0;
         buffer = NULL;
+        ret = AVERROR_INVALIDDATA;
     }
 
+    buffer_size = (bit_size + 7) >> 3;
+
     s->buffer       = buffer;
     s->size_in_bits = bit_size;
 #if !UNCHECKED_BITSTREAM_READER
@@ -383,6 +390,7 @@ static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer,
 #endif
     s->buffer_end   = buffer + buffer_size;
     s->index        = 0;
+    return ret;
 }
 
 static inline void align_get_bits(GetBitContext *s)