Merge commit '3c27275c1309190f2d6ed69140b67d014215b6c9'

* commit '3c27275c1309190f2d6ed69140b67d014215b6c9':
  tiff: Check the check_size() return value and forward it

Conflicts:
	libavcodec/tiffenc.c

See: d50aa006fb3430bedc3872ba10e028a714499625
Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 40 insertions, 23 deletions

diff --git a/libavcodec/tiffenc.c b/libavcodec/tiffenc.c
index 72219da0d0..782a2fbfd7 100644
--- a/libavcodec/tiffenc.c
+++ b/libavcodec/tiffenc.c
@@ -123,8 +123,8 @@ static void tnput(uint8_t **p, int n, const uint8_t *val, enum TiffTypes type,
  * @param count the number of values
  * @param ptr_val pointer to values
  */
-static void add_entry(TiffEncoderContext *s, enum TiffTags tag,
-                      enum TiffTypes type, int count, const void *ptr_val)
+static int add_entry(TiffEncoderContext *s, enum TiffTags tag,
+                     enum TiffTypes type, int count, const void *ptr_val)
 {
     uint8_t *entries_ptr = s->entries + 12 * s->num_entries;
 
@@ -138,19 +138,22 @@ static void add_entry(TiffEncoderContext *s, enum TiffTags tag,
         tnput(&entries_ptr, count, ptr_val, type, 0);
     } else {
         bytestream_put_le32(&entries_ptr, *s->buf - s->buf_start);
-        check_size(s, count * (int64_t)type_sizes2[type]);
+        if (check_size(s, count * (int64_t)type_sizes2[type]))
+            return AVERROR_INVALIDDATA;
         tnput(s->buf, count, ptr_val, type, 0);
     }
 
     s->num_entries++;
+    return 0;
 }
 
-static void add_entry1(TiffEncoderContext *s,
-                       enum TiffTags tag, enum TiffTypes type, int val)
+static int add_entry1(TiffEncoderContext *s,
+                      enum TiffTags tag, enum TiffTypes type, int val)
 {
     uint16_t w  = val;
     uint32_t dw = val;
-    add_entry(s, tag, type, 1, type == TIFF_SHORT ? (void *)&w : (void *)&dw);
+    return add_entry(s, tag, type, 1,
+                     type == TIFF_SHORT ? (void *)&w : (void *)&dw);
 }
 
 /**
@@ -222,6 +225,20 @@ static void pack_yuv(TiffEncoderContext *s, const AVFrame *p,
     }
 }
 
+#define ADD_ENTRY(s, tag, type, count, ptr_val)         \
+    do {                                                \
+        ret = add_entry(s, tag, type, count, ptr_val);  \
+        if (ret < 0)                                    \
+            goto fail;                                  \
+    } while(0);
+
+#define ADD_ENTRY1(s, tag, type, val)           \
+    do {                                        \
+        ret = add_entry1(s, tag, type, val);    \
+        if (ret < 0)                            \
+            goto fail;                          \
+    } while(0);
+
 static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                         const AVFrame *pict, int *got_packet)
 {
@@ -425,23 +442,23 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 
     s->num_entries = 0;
 
-    add_entry1(s, TIFF_SUBFILE, TIFF_LONG, 0);
-    add_entry1(s, TIFF_WIDTH,   TIFF_LONG, s->width);
-    add_entry1(s, TIFF_HEIGHT,  TIFF_LONG, s->height);
+    ADD_ENTRY1(s, TIFF_SUBFILE, TIFF_LONG, 0);
+    ADD_ENTRY1(s, TIFF_WIDTH,   TIFF_LONG, s->width);
+    ADD_ENTRY1(s, TIFF_HEIGHT,  TIFF_LONG, s->height);
 
     if (s->bpp_tab_size)
-        add_entry(s, TIFF_BPP, TIFF_SHORT, s->bpp_tab_size, bpp_tab);
+        ADD_ENTRY(s, TIFF_BPP, TIFF_SHORT, s->bpp_tab_size, bpp_tab);
 
-    add_entry1(s, TIFF_COMPR,       TIFF_SHORT, s->compr);
-    add_entry1(s, TIFF_PHOTOMETRIC, TIFF_SHORT, s->photometric_interpretation);
-    add_entry(s,  TIFF_STRIP_OFFS,  TIFF_LONG,  strips, s->strip_offsets);
+    ADD_ENTRY1(s, TIFF_COMPR,       TIFF_SHORT, s->compr);
+    ADD_ENTRY1(s, TIFF_PHOTOMETRIC, TIFF_SHORT, s->photometric_interpretation);
+    ADD_ENTRY(s,  TIFF_STRIP_OFFS,  TIFF_LONG,  strips, s->strip_offsets);
 
     if (s->bpp_tab_size)
-        add_entry1(s, TIFF_SAMPLES_PER_PIXEL, TIFF_SHORT, s->bpp_tab_size);
+        ADD_ENTRY1(s, TIFF_SAMPLES_PER_PIXEL, TIFF_SHORT, s->bpp_tab_size);
 
-    add_entry1(s, TIFF_ROWSPERSTRIP, TIFF_LONG,     s->rps);
-    add_entry(s,  TIFF_STRIP_SIZE,   TIFF_LONG,     strips, s->strip_sizes);
-    add_entry(s,  TIFF_XRES,         TIFF_RATIONAL, 1,      res);
+    ADD_ENTRY1(s, TIFF_ROWSPERSTRIP, TIFF_LONG,     s->rps);
+    ADD_ENTRY(s,  TIFF_STRIP_SIZE,   TIFF_LONG,     strips, s->strip_sizes);
+    ADD_ENTRY(s,  TIFF_XRES,         TIFF_RATIONAL, 1,      res);
     if (avctx->sample_aspect_ratio.num > 0 &&
         avctx->sample_aspect_ratio.den > 0) {
         AVRational y = av_mul_q(av_make_q(s->dpi, 1),
@@ -449,11 +466,11 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         res[0] = y.num;
         res[1] = y.den;
     }
-    add_entry(s,  TIFF_YRES,         TIFF_RATIONAL, 1,      res);
-    add_entry1(s, TIFF_RES_UNIT,     TIFF_SHORT,    2);
+    ADD_ENTRY(s,  TIFF_YRES,         TIFF_RATIONAL, 1,      res);
+    ADD_ENTRY1(s, TIFF_RES_UNIT,     TIFF_SHORT,    2);
 
     if (!(avctx->flags & CODEC_FLAG_BITEXACT))
-        add_entry(s, TIFF_SOFTWARE_NAME, TIFF_STRING,
+        ADD_ENTRY(s, TIFF_SOFTWARE_NAME, TIFF_STRING,
                   strlen(LIBAVCODEC_IDENT) + 1, LIBAVCODEC_IDENT);
 
     if (avctx->pix_fmt == AV_PIX_FMT_PAL8) {
@@ -464,17 +481,17 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             pal[i + 256] = ((rgb >>  8) & 0xff) * 257;
             pal[i + 512] =  (rgb        & 0xff) * 257;
         }
-        add_entry(s, TIFF_PAL, TIFF_SHORT, 256 * 3, pal);
+        ADD_ENTRY(s, TIFF_PAL, TIFF_SHORT, 256 * 3, pal);
     }
     if (alpha)
         add_entry1(s,TIFF_EXTRASAMPLES,      TIFF_SHORT,            2);
     if (is_yuv) {
         /** according to CCIR Recommendation 601.1 */
         uint32_t refbw[12] = { 15, 1, 235, 1, 128, 1, 240, 1, 128, 1, 240, 1 };
-        add_entry(s, TIFF_YCBCR_SUBSAMPLING, TIFF_SHORT,    2, s->subsampling);
+        ADD_ENTRY(s, TIFF_YCBCR_SUBSAMPLING, TIFF_SHORT,    2, s->subsampling);
         if (avctx->chroma_sample_location == AVCHROMA_LOC_TOPLEFT)
             add_entry1(s, TIFF_YCBCR_POSITIONING, TIFF_SHORT, 2);
-        add_entry(s, TIFF_REFERENCE_BW,      TIFF_RATIONAL, 6, refbw);
+        ADD_ENTRY(s, TIFF_REFERENCE_BW,      TIFF_RATIONAL, 6, refbw);
     }
     // write offset to dir
     bytestream_put_le32(&offset, ptr - pkt->data);