Merge remote-tracking branch 'qatar/release/0.6' into release/0.6

* qatar/release/0.6:
  vorbis: Validate that the floor 1 X values contain no duplicates.
  lavfi: avfilter_merge_formats: handle case where inputs are same
  mpegvideo: Don't use ff_mspel_motion() for vc1
  imgconvert: avoid undefined left shift in avcodec_find_best_pix_fmt
  nuv: check RTjpeg header for validity
  vc1dec: add flush function for WMV9 and VC-1 decoders
  mov: set AVCodecContext.width/height for h264
  h264: allow cropping to AVCodecContext.width/height

Merged-by: Michael Niedermayer <michaelni@gmx.at>

12 files changed, 40 insertions, 13 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 06ded4d1f1..02ff83309f 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1799,6 +1799,12 @@ static int decode_slice_header(H264Context *h, H264Context *h0){
     else
         s->height= 16*s->mb_height - 4*FFMIN(h->sps.crop_bottom, 3);
 
+    if (FFALIGN(s->avctx->width,  16) == s->width &&
+        FFALIGN(s->avctx->height, 16) == s->height) {
+        s->width  = s->avctx->width;
+        s->height = s->avctx->height;
+    }
+
     if (s->context_initialized
         && (   s->width != s->avctx->width || s->height != s->avctx->height
             || av_cmp_q(h->sps.sar, s->avctx->sample_aspect_ratio))) {
diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index 8f789c4ae0..ae8ec16543 100644
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -851,7 +851,8 @@ static enum PixelFormat avcodec_find_best_pix_fmt1(int64_t pix_fmt_mask,
     /* find exact color match with smallest size */
     dst_pix_fmt = PIX_FMT_NONE;
     min_dist = 0x7fffffff;
-    for(i = 0;i < PIX_FMT_NB; i++) {
+    /* test only the first 64 pixel formats to avoid undefined behaviour */
+    for (i = 0; i < 64; i++) {
         if (pix_fmt_mask & (1ULL << i)) {
             loss = avcodec_get_pix_fmt_loss(i, src_pix_fmt, has_alpha) & loss_mask;
             if (loss == 0) {
diff --git a/libavcodec/mpegvideo_common.h b/libavcodec/mpegvideo_common.h
index 73106664f9..258f7cdaa7 100644
--- a/libavcodec/mpegvideo_common.h
+++ b/libavcodec/mpegvideo_common.h
@@ -727,7 +727,8 @@ static av_always_inline void MPV_motion_internal(MpegEncContext *s,
                         0, 0, 0,
                         ref_picture, pix_op, qpix_op,
                         s->mv[dir][0][0], s->mv[dir][0][1], 16);
-        }else if(!is_mpeg12 && (CONFIG_WMV2_DECODER || CONFIG_WMV2_ENCODER) && s->mspel){
+        } else if (!is_mpeg12 && (CONFIG_WMV2_DECODER || CONFIG_WMV2_ENCODER) &&
+                   s->mspel && s->codec_id == CODEC_ID_WMV2) {
             ff_mspel_motion(s, dest_y, dest_cb, dest_cr,
                         ref_picture, pix_op,
                         s->mv[dir][0][0], s->mv[dir][0][1], 16);
diff --git a/libavcodec/nuv.c b/libavcodec/nuv.c
index 791f450913..6f832cc27a 100644
--- a/libavcodec/nuv.c
+++ b/libavcodec/nuv.c
@@ -184,17 +184,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     }
     if (c->codec_frameheader) {
         int w, h, q;
-        if (buf_size < 12) {
+        if (buf_size < RTJPEG_HEADER_SIZE || buf[4] != RTJPEG_HEADER_SIZE ||
+            buf[5] != RTJPEG_FILE_VERSION) {
             av_log(avctx, AV_LOG_ERROR, "invalid nuv video frame\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         w = AV_RL16(&buf[6]);
         h = AV_RL16(&buf[8]);
         q = buf[10];
         if (!codec_reinit(avctx, w, h, q))
             return -1;
-        buf = &buf[12];
-        buf_size -= 12;
+        buf = &buf[RTJPEG_HEADER_SIZE];
+        buf_size -= RTJPEG_HEADER_SIZE;
     }
 
     if (keyframe && c->pic.data[0])
diff --git a/libavcodec/rtjpeg.h b/libavcodec/rtjpeg.h
index 4bcb9f70ca..73d41f481d 100644
--- a/libavcodec/rtjpeg.h
+++ b/libavcodec/rtjpeg.h
@@ -25,6 +25,9 @@
 #include <stdint.h>
 #include "dsputil.h"
 
+#define RTJPEG_FILE_VERSION 0
+#define RTJPEG_HEADER_SIZE 12
+
 typedef struct {
     int w, h;
     DSPContext *dsp;
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index abcc5ecda8..8d31e8bc1e 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -3328,6 +3328,7 @@ AVCodec vc1_decoder = {
     vc1_decode_frame,
     CODEC_CAP_DR1 | CODEC_CAP_DELAY,
     NULL,
+    .flush          = ff_mpeg_flush,
     .long_name = NULL_IF_CONFIG_SMALL("SMPTE VC-1"),
     .pix_fmts = ff_hwaccel_pixfmt_list_420
 };
@@ -3344,6 +3345,7 @@ AVCodec wmv3_decoder = {
     vc1_decode_frame,
     CODEC_CAP_DR1 | CODEC_CAP_DELAY,
     NULL,
+    .flush          = ff_mpeg_flush,
     .long_name = NULL_IF_CONFIG_SMALL("Windows Media Video 9"),
     .pix_fmts = ff_hwaccel_pixfmt_list_420
 };
diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index 109737976c..5f0ddda32f 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -123,7 +123,8 @@ int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, uint_fast32_t num)
     return 0;
 }
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values)
 {
     int i;
     list[0].sort = 0;
@@ -147,6 +148,11 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     for (i = 0; i < values - 1; i++) {
         int j;
         for (j = i + 1; j < values; j++) {
+            if (list[i].x == list[j].x) {
+                av_log(avccontext, AV_LOG_ERROR,
+                       "Duplicate value found in floor 1 X coordinates\n");
+                return AVERROR_INVALIDDATA;
+            }
             if (list[list[i].sort].x > list[list[j].sort].x) {
                 int tmp = list[i].sort;
                 list[i].sort = list[j].sort;
@@ -154,6 +160,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
             }
         }
     }
+    return 0;
 }
 
 static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
diff --git a/libavcodec/vorbis.h b/libavcodec/vorbis.h
index ce9bead425..18a826eee8 100644
--- a/libavcodec/vorbis.h
+++ b/libavcodec/vorbis.h
@@ -35,7 +35,8 @@ typedef struct {
     uint_fast16_t high;
 } vorbis_floor1_entry;
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values);
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values);
 unsigned int ff_vorbis_nth_root(unsigned int x, unsigned int n); // x^(1/n)
 int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, uint_fast32_t num);
 void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
diff --git a/libavcodec/vorbis_dec.c b/libavcodec/vorbis_dec.c
index e5ad4aaae3..a2ae854699 100644
--- a/libavcodec/vorbis_dec.c
+++ b/libavcodec/vorbis_dec.c
@@ -550,7 +550,11 @@ static int vorbis_parse_setup_hdr_floors(vorbis_context *vc)
             }
 
 // Precalculate order of x coordinates - needed for decode
-            ff_vorbis_ready_floor1_list(floor_setup->data.t1.list, floor_setup->data.t1.x_list_dim);
+            if (ff_vorbis_ready_floor1_list(vc->avccontext,
+                                            floor_setup->data.t1.list,
+                                            floor_setup->data.t1.x_list_dim)) {
+                return AVERROR_INVALIDDATA;
+            }
         } else if (floor_setup->floor_type == 0) {
             uint_fast8_t max_codebook_dim = 0;
 
diff --git a/libavcodec/vorbis_enc.c b/libavcodec/vorbis_enc.c
index 934463d1e1..2fb41d16eb 100644
--- a/libavcodec/vorbis_enc.c
+++ b/libavcodec/vorbis_enc.c
@@ -302,7 +302,8 @@ static void create_vorbis_context(vorbis_enc_context *venc,
         };
         fc->list[i].x = a[i - 2];
     }
-    ff_vorbis_ready_floor1_list(fc->list, fc->values);
+    if (ff_vorbis_ready_floor1_list(avccontext, fc->list, fc->values))
+        return AVERROR(EINVAL);
 
     venc->nresidues = 1;
     venc->residues  = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues);
diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 2a9bdb0bd0..8b8bf75962 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -44,6 +44,9 @@ AVFilterFormats *avfilter_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
     AVFilterFormats *ret;
     unsigned i, j, k = 0;
 
+    if (a == b)
+        return a;
+
     ret = av_mallocz(sizeof(AVFilterFormats));
 
     /* merge list of formats */
diff --git a/libavformat/mov.c b/libavformat/mov.c
index 2edd27e440..f1cde0b767 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -1820,9 +1820,6 @@ static int mov_read_trak(MOVContext *c, ByteIOContext *pb, MOVAtom atom)
 #if CONFIG_H263_DECODER
     case CODEC_ID_H263:
 #endif
-#if CONFIG_H264_DECODER
-    case CODEC_ID_H264:
-#endif
 #if CONFIG_MPEG4_DECODER
     case CODEC_ID_MPEG4:
 #endif