Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5:
  Fix memory (re)allocation in matroskadec.c, related to MSVR-11-0080.
  cavs: fix some crashes with invalid bitstreams
  mjpeg: Detect overreads in mjpeg_decode_scan() and error out.

Merged-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 41 insertions, 11 deletions

diff --git a/libavcodec/cavsdec.c b/libavcodec/cavsdec.c
index a1895bcb88..8d30040d74 100644
--- a/libavcodec/cavsdec.c
+++ b/libavcodec/cavsdec.c
@@ -130,12 +130,14 @@ static int decode_residual_block(AVSContext *h, GetBitContext *gb,
                 r++;
             mask = -(level_code & 1);
             level = (level^mask) - mask;
-        } else {
+        } else if (level_code >= 0) {
             level = r->rltab[level_code][0];
             if(!level) //end of block signal
                 break;
             run   = r->rltab[level_code][1];
             r += r->rltab[level_code][2];
+        } else {
+            break;
         }
         level_buf[i] = level;
         run_buf[i] = run;
@@ -189,7 +191,8 @@ static inline int decode_residual_inter(AVSContext *h) {
 
 static int decode_mb_i(AVSContext *h, int cbp_code) {
     GetBitContext *gb = &h->s.gb;
-    int block, pred_mode_uv;
+    unsigned pred_mode_uv;
+    int block;
     uint8_t top[18];
     uint8_t *left = NULL;
     uint8_t *d;
@@ -445,6 +448,8 @@ static inline int check_for_slice(AVSContext *h) {
     if((show_bits_long(gb,24+align) & 0xFFFFFF) == 0x000001) {
         skip_bits_long(gb,24+align);
         h->stc = get_bits(gb,8);
+        if (h->stc >= h->mb_height)
+            return 0;
         decode_slice_header(h,gb);
         return 1;
     }
@@ -648,7 +653,7 @@ static int cavs_decode_frame(AVCodecContext * avctx,void *data, int *data_size,
     buf_end = buf + buf_size;
     for(;;) {
         buf_ptr = ff_find_start_code(buf_ptr,buf_end, &stc);
-        if(stc & 0xFFFFFE00)
+        if((stc & 0xFFFFFE00) || buf_ptr == buf_end)
             return FFMAX(0, buf_ptr - buf - s->parse_context.last_index);
         input_size = (buf_end - buf_ptr)*8;
         switch(stc) {
diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index 145719c16e..e5c9f38b93 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -784,6 +784,10 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah, i
             if (s->restart_interval && !s->restart_count)
                 s->restart_count = s->restart_interval;
 
+            if(get_bits_count(&s->gb)>s->gb.size_in_bits){
+                av_log(s->avctx, AV_LOG_ERROR, "overread %d\n", get_bits_count(&s->gb) - s->gb.size_in_bits);
+                return -1;
+            }
             for(i=0;i<nb_components;i++) {
                 uint8_t *ptr;
                 int n, h, v, x, y, c, j;
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index ac12f1ea66..d9ffec3727 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -752,11 +752,15 @@ static int ebml_parse_elem(MatroskaDemuxContext *matroska,
     uint32_t id = syntax->id;
     uint64_t length;
     int res;
+    void *newelem;
 
     data = (char *)data + syntax->data_offset;
     if (syntax->list_elem_size) {
         EbmlList *list = data;
-        list->elem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        newelem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        if (!newelem)
+            return AVERROR(ENOMEM);
+        list->elem = newelem;
         data = (char*)list->elem + list->nb_elem*syntax->list_elem_size;
         memset(data, 0, syntax->list_elem_size);
         list->nb_elem++;
@@ -873,6 +877,7 @@ static int matroska_decode_buffer(uint8_t** buf, int* buf_size,
     uint8_t* data = *buf;
     int isize = *buf_size;
     uint8_t* pkt_data = NULL;
+    uint8_t* newpktdata;
     int pkt_size = isize;
     int result = 0;
     int olen;
@@ -899,7 +904,12 @@ static int matroska_decode_buffer(uint8_t** buf, int* buf_size,
         zstream.avail_in = isize;
         do {
             pkt_size *= 3;
-            pkt_data = av_realloc(pkt_data, pkt_size);
+            newpktdata = av_realloc(pkt_data, pkt_size);
+            if (!newpktdata) {
+                inflateEnd(&zstream);
+                goto failed;
+            }
+            pkt_data = newpktdata;
             zstream.avail_out = pkt_size - zstream.total_out;
             zstream.next_out = pkt_data + zstream.total_out;
             result = inflate(&zstream, Z_NO_FLUSH);
@@ -920,7 +930,12 @@ static int matroska_decode_buffer(uint8_t** buf, int* buf_size,
         bzstream.avail_in = isize;
         do {
             pkt_size *= 3;
-            pkt_data = av_realloc(pkt_data, pkt_size);
+            newpktdata = av_realloc(pkt_data, pkt_size);
+            if (!newpktdata) {
+                BZ2_bzDecompressEnd(&bzstream);
+                goto failed;
+            }
+            pkt_data = newpktdata;
             bzstream.avail_out = pkt_size - bzstream.total_out_lo32;
             bzstream.next_out = pkt_data + bzstream.total_out_lo32;
             result = BZ2_bzDecompress(&bzstream);
@@ -975,13 +990,17 @@ static void matroska_fix_ass_packet(MatroskaDemuxContext *matroska,
     }
 }
 
-static void matroska_merge_packets(AVPacket *out, AVPacket *in)
+static int matroska_merge_packets(AVPacket *out, AVPacket *in)
 {
-    out->data = av_realloc(out->data, out->size+in->size);
+    void *newdata = av_realloc(out->data, out->size+in->size);
+    if (!newdata)
+        return AVERROR(ENOMEM);
+    out->data = newdata;
     memcpy(out->data+out->size, in->data, in->size);
     out->size += in->size;
     av_destruct_packet(in);
     av_free(in);
+    return 0;
 }
 
 static void matroska_convert_tag(AVFormatContext *s, EbmlList *list,
@@ -1462,11 +1481,13 @@ static int matroska_deliver_packet(MatroskaDemuxContext *matroska,
         memcpy(pkt, matroska->packets[0], sizeof(AVPacket));
         av_free(matroska->packets[0]);
         if (matroska->num_packets > 1) {
+            void *newpackets;
             memmove(&matroska->packets[0], &matroska->packets[1],
                     (matroska->num_packets - 1) * sizeof(AVPacket *));
-            matroska->packets =
-                av_realloc(matroska->packets, (matroska->num_packets - 1) *
-                           sizeof(AVPacket *));
+            newpackets = av_realloc(matroska->packets,
+                            (matroska->num_packets - 1) * sizeof(AVPacket *));
+            if (newpackets)
+                matroska->packets = newpackets;
         } else {
             av_freep(&matroska->packets);
         }