Use a shared function to validate FLAC extradata.

Originally committed as revision 17602 to svn://svn.ffmpeg.org/ffmpeg/trunk

6 files changed, 91 insertions, 31 deletions

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index afb1ede3fe..afa5fac152 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -345,15 +345,16 @@ OBJS-$(CONFIG_ADPCM_YAMAHA_ENCODER)       += adpcm.o
 
 # libavformat dependencies
 OBJS-$(CONFIG_EAC3_DEMUXER)            += ac3_parser.o ac3tab.o aac_ac3_parser.o
+OBJS-$(CONFIG_FLAC_MUXER)              += flacdec.o
 OBJS-$(CONFIG_GXF_DEMUXER)             += mpeg12data.o
-OBJS-$(CONFIG_MATROSKA_AUDIO_MUXER)    += xiph.o mpeg4audio.o
+OBJS-$(CONFIG_MATROSKA_AUDIO_MUXER)    += xiph.o mpeg4audio.o flacdec.o
 OBJS-$(CONFIG_MATROSKA_DEMUXER)        += mpeg4audio.o
-OBJS-$(CONFIG_MATROSKA_MUXER)          += xiph.o mpeg4audio.o
+OBJS-$(CONFIG_MATROSKA_MUXER)          += xiph.o mpeg4audio.o flacdec.o
 OBJS-$(CONFIG_MOV_DEMUXER)             += mpeg4audio.o mpegaudiodata.o
 OBJS-$(CONFIG_MPEGTS_MUXER)            += mpegvideo.o
 OBJS-$(CONFIG_NUT_MUXER)               += mpegaudiodata.o
 OBJS-$(CONFIG_OGG_DEMUXER)             += flacdec.o
-OBJS-$(CONFIG_OGG_MUXER)               += xiph.o
+OBJS-$(CONFIG_OGG_MUXER)               += xiph.o flacdec.o
 OBJS-$(CONFIG_RTP_MUXER)               += mpegvideo.o
 
 # external codec libraries
diff --git a/libavcodec/flac.h b/libavcodec/flac.h
index 9a4f820831..8af79f2bd8 100644
--- a/libavcodec/flac.h
+++ b/libavcodec/flac.h
@@ -42,6 +42,11 @@ enum {
     FLAC_METADATA_TYPE_INVALID = 127
 };
 
+enum FLACExtradataFormat {
+    FLAC_EXTRADATA_FORMAT_STREAMINFO  = 0,
+    FLAC_EXTRADATA_FORMAT_FULL_HEADER = 1
+};
+
 /**
  * Data needed from the Streaminfo header for use by the raw FLAC demuxer
  * and/or the FLAC decoder.
@@ -68,4 +73,15 @@ typedef struct FLACStreaminfo {
 void ff_flac_parse_streaminfo(AVCodecContext *avctx, struct FLACStreaminfo *s,
                               const uint8_t *buffer);
 
+/**
+ * Validate the FLAC extradata.
+ * @param[in]  avctx codec context containing the extradata.
+ * @param[out] format extradata format.
+ * @param[out] streaminfo_start pointer to start of 34-byte STREAMINFO data.
+ * @return 1 if valid, 0 if not valid.
+ */
+int ff_flac_is_extradata_valid(AVCodecContext *avctx,
+                               enum FLACExtradataFormat *format,
+                               uint8_t **streaminfo_start);
+
 #endif /* AVCODEC_FLAC_H */
diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c
index d6e0c8620f..42310b24a0 100644
--- a/libavcodec/flacdec.c
+++ b/libavcodec/flacdec.c
@@ -96,26 +96,55 @@ static int64_t get_utf8(GetBitContext *gb)
 }
 
 static void allocate_buffers(FLACContext *s);
-static int metadata_parse(FLACContext *s);
+
+int ff_flac_is_extradata_valid(AVCodecContext *avctx,
+                               enum FLACExtradataFormat *format,
+                               uint8_t **streaminfo_start)
+{
+    if (!avctx->extradata || avctx->extradata_size < FLAC_STREAMINFO_SIZE) {
+        av_log(avctx, AV_LOG_ERROR, "extradata NULL or too small.\n");
+        return 0;
+    }
+    if (AV_RL32(avctx->extradata) != MKTAG('f','L','a','C')) {
+        /* extradata contains STREAMINFO only */
+        if (avctx->extradata_size != FLAC_STREAMINFO_SIZE) {
+            av_log(avctx, AV_LOG_WARNING, "extradata contains %d bytes too many.\n",
+                   FLAC_STREAMINFO_SIZE-avctx->extradata_size);
+        }
+        *format = FLAC_EXTRADATA_FORMAT_STREAMINFO;
+        *streaminfo_start = avctx->extradata;
+    } else {
+        if (avctx->extradata_size < 8+FLAC_STREAMINFO_SIZE) {
+            av_log(avctx, AV_LOG_ERROR, "extradata too small.\n");
+            return 0;
+        }
+        *format = FLAC_EXTRADATA_FORMAT_FULL_HEADER;
+        *streaminfo_start = &avctx->extradata[8];
+    }
+    return 1;
+}
 
 static av_cold int flac_decode_init(AVCodecContext *avctx)
 {
+    enum FLACExtradataFormat format;
+    uint8_t *streaminfo;
     FLACContext *s = avctx->priv_data;
     s->avctx = avctx;
 
     avctx->sample_fmt = SAMPLE_FMT_S16;
 
-    if (avctx->extradata_size > 4) {
+    /* for now, the raw FLAC header is allowed to be passed to the decoder as
+       frame data instead of extradata. */
+    if (!avctx->extradata)
+        return 0;
+
+    if (!ff_flac_is_extradata_valid(avctx, &format, &streaminfo))
+        return -1;
+
         /* initialize based on the demuxer-supplied streamdata header */
-        if (avctx->extradata_size == FLAC_STREAMINFO_SIZE) {
             ff_flac_parse_streaminfo(avctx, (FLACStreaminfo *)s,
-                                     avctx->extradata);
+                                     streaminfo);
             allocate_buffers(s);
-        } else {
-            init_get_bits(&s->gb, avctx->extradata, avctx->extradata_size*8);
-            metadata_parse(s);
-        }
-    }
 
     return 0;
 }
diff --git a/libavformat/flacenc.c b/libavformat/flacenc.c
index 093a07a823..5595d76b7d 100644
--- a/libavformat/flacenc.c
+++ b/libavformat/flacenc.c
@@ -19,6 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "libavcodec/flac.h"
 #include "avformat.h"
 
 static int flac_write_header(struct AVFormatContext *s)
@@ -26,9 +27,15 @@ static int flac_write_header(struct AVFormatContext *s)
     static const uint8_t header[8] = {
         0x66, 0x4C, 0x61, 0x43, 0x80, 0x00, 0x00, 0x22
     };
-    uint8_t *streaminfo = s->streams[0]->codec->extradata;
+    AVCodecContext *codec = s->streams[0]->codec;
+    uint8_t *streaminfo;
     int len = s->streams[0]->codec->extradata_size;
-    if(streaminfo != NULL && len > 0) {
+    enum FLACExtradataFormat format;
+
+    if (!ff_flac_is_extradata_valid(codec, &format, &streaminfo))
+        return -1;
+
+    if (format == FLAC_EXTRADATA_FORMAT_STREAMINFO) {
         put_buffer(s->pb, header, 8);
         put_buffer(s->pb, streaminfo, len);
     }
@@ -38,16 +45,22 @@ static int flac_write_header(struct AVFormatContext *s)
 static int flac_write_trailer(struct AVFormatContext *s)
 {
     ByteIOContext *pb = s->pb;
-    uint8_t *streaminfo = s->streams[0]->codec->extradata;
-    int len = s->streams[0]->codec->extradata_size;
+    uint8_t *streaminfo;
+    enum FLACExtradataFormat format;
     int64_t file_size;
 
-    if (streaminfo && len > 0 && !url_is_streamed(s->pb)) {
+    if (!ff_flac_is_extradata_valid(s->streams[0]->codec, &format, &streaminfo))
+        return -1;
+
+    if (!url_is_streamed(pb)) {
+        /* rewrite the STREAMINFO header block data */
         file_size = url_ftell(pb);
         url_fseek(pb, 8, SEEK_SET);
-        put_buffer(pb, streaminfo, len);
+        put_buffer(pb, streaminfo, FLAC_STREAMINFO_SIZE);
         url_fseek(pb, file_size, SEEK_SET);
         put_flush_packet(pb);
+    } else {
+        av_log(s, AV_LOG_WARNING, "unable to rewrite FLAC header.\n");
     }
     return 0;
 }
diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 44787be2d2..0b0adef1ba 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -28,6 +28,7 @@
 #include "libavutil/md5.h"
 #include "libavcodec/xiph.h"
 #include "libavcodec/mpeg4audio.h"
+#include "libavcodec/flac.h"
 
 typedef struct ebml_master {
     int64_t         pos;                ///< absolute offset in the file where the master's elements start
@@ -420,23 +421,20 @@ static int put_xiph_codecpriv(AVFormatContext *s, ByteIOContext *pb, AVCodecCont
     return 0;
 }
 
-#define FLAC_STREAMINFO_SIZE 34
-
 static int put_flac_codecpriv(AVFormatContext *s, ByteIOContext *pb, AVCodecContext *codec)
 {
-    // if the extradata_size is greater than FLAC_STREAMINFO_SIZE,
-    // assume that it's in Matroska format already
-    if (codec->extradata_size < FLAC_STREAMINFO_SIZE) {
+    uint8_t *streaminfo;
+    enum FLACExtradataFormat format;
+
+    if (!ff_flac_is_extradata_valid(codec, &format, &streaminfo)) {
         av_log(s, AV_LOG_ERROR, "Invalid FLAC extradata\n");
         return -1;
-    } else if (codec->extradata_size == FLAC_STREAMINFO_SIZE) {
+    }
+    if (format == FLAC_EXTRADATA_FORMAT_STREAMINFO) {
         // only the streaminfo packet
         put_buffer(pb, "fLaC", 4);
         put_byte(pb, 0x80);
         put_be24(pb, FLAC_STREAMINFO_SIZE);
-    } else if(memcmp("fLaC", codec->extradata, 4)) {
-        av_log(s, AV_LOG_ERROR, "Invalid FLAC extradata\n");
-        return -1;
     }
     put_buffer(pb, codec->extradata, codec->extradata_size);
     return 0;
diff --git a/libavformat/oggenc.c b/libavformat/oggenc.c
index 4fdae05c8f..bbbee0a906 100644
--- a/libavformat/oggenc.c
+++ b/libavformat/oggenc.c
@@ -22,6 +22,7 @@
 #include "libavutil/crc.h"
 #include "libavcodec/xiph.h"
 #include "libavcodec/bytestream.h"
+#include "libavcodec/flac.h"
 #include "avformat.h"
 #include "internal.h"
 
@@ -82,12 +83,14 @@ static int ogg_write_page(AVFormatContext *s, const uint8_t *data, int size,
     return size;
 }
 
-static int ogg_build_flac_headers(const uint8_t *extradata, int extradata_size,
+static int ogg_build_flac_headers(AVCodecContext *avctx,
                                   OGGStreamContext *oggstream, int bitexact)
 {
     const char *vendor = bitexact ? "ffmpeg" : LIBAVFORMAT_IDENT;
+    enum FLACExtradataFormat format;
+    uint8_t *streaminfo;
     uint8_t *p;
-    if (extradata_size != 34)
+    if (!ff_flac_is_extradata_valid(avctx, &format, &streaminfo))
         return -1;
     oggstream->header_len[0] = 51;
     oggstream->header[0] = av_mallocz(51); // per ogg flac specs
@@ -100,7 +103,7 @@ static int ogg_build_flac_headers(const uint8_t *extradata, int extradata_size,
     bytestream_put_buffer(&p, "fLaC", 4);
     bytestream_put_byte(&p, 0x00); // streaminfo
     bytestream_put_be24(&p, 34);
-    bytestream_put_buffer(&p, extradata, 34);
+    bytestream_put_buffer(&p, streaminfo, FLAC_STREAMINFO_SIZE);
     oggstream->header_len[1] = 1+3+4+strlen(vendor)+4;
     oggstream->header[1] = av_mallocz(oggstream->header_len[1]);
     p = oggstream->header[1];
@@ -136,7 +139,7 @@ static int ogg_write_header(AVFormatContext *s)
         oggstream = av_mallocz(sizeof(*oggstream));
         st->priv_data = oggstream;
         if (st->codec->codec_id == CODEC_ID_FLAC) {
-            if (ogg_build_flac_headers(st->codec->extradata, st->codec->extradata_size,
+            if (ogg_build_flac_headers(st->codec,
                                        oggstream, st->codec->flags & CODEC_FLAG_BITEXACT) < 0) {
                 av_log(s, AV_LOG_ERROR, "Extradata corrupted\n");
                 av_freep(&st->priv_data);