dnxhd: add better support for CIDs 1270 to 1274

These are DNxHR profiles with the following properties:
- Variable size in a profile (property added in a previous commit),
  requiring variable-sized macroblock table;
- Variable bitdepth, up to 12 bits.
- Better validation of buffer sizes and positions

Signed-off-by: Christophe Gisquet <christophe.gisquet@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 71 insertions, 21 deletions

diff --git a/libavcodec/dnxhddec.c b/libavcodec/dnxhddec.c
index 147143963c..fec9aac7e2 100644
--- a/libavcodec/dnxhddec.c
+++ b/libavcodec/dnxhddec.c
@@ -55,13 +55,14 @@ typedef struct DNXHDContext {
     unsigned int width, height;
     enum AVPixelFormat pix_fmt;
     unsigned int mb_width, mb_height;
-    uint32_t mb_scan_index[68];         /* max for 1080p */
+    uint32_t *mb_scan_index;
+    int data_offset;                    // End of mb_scan_index, where macroblocks start
     int cur_field;                      ///< current interlaced field
     VLC ac_vlc, dc_vlc, run_vlc;
     IDCTDSPContext idsp;
     ScanTable scantable;
     const CIDEntry *cid_table;
-    int bit_depth; // 8, 10 or 0 if not initialized at all.
+    int bit_depth; // 8, 10, 12 or 0 if not initialized at all.
     int is_444;
     int mbaff;
     int act;
@@ -78,6 +79,10 @@ static int dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
                                      RowContext *row, int n);
 static int dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
                                          RowContext *row, int n);
+static int dnxhd_decode_dct_block_12(const DNXHDContext *ctx,
+                                     RowContext *row, int n);
+static int dnxhd_decode_dct_block_12_444(const DNXHDContext *ctx,
+                                         RowContext *row, int n);
 
 static av_cold int dnxhd_decode_init(AVCodecContext *avctx)
 {
@@ -153,9 +158,11 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
 {
     static const uint8_t header_prefix[]    = { 0x00, 0x00, 0x02, 0x80, 0x01 };
     static const uint8_t header_prefix444[] = { 0x00, 0x00, 0x02, 0x80, 0x02 };
-    static const uint8_t header_prefixhr[] = { 0x00, 0x00, 0x02, 0x80, 0x03 };
+    static const uint8_t header_prefixhr1[] = { 0x00, 0x00, 0x02, 0x80, 0x03 };
+    static const uint8_t header_prefixhr2[] = { 0x00, 0x00, 0x03, 0x8C, 0x03 };
     int i, cid, ret;
     int old_bit_depth = ctx->bit_depth;
+    int old_mb_height = ctx->mb_height;
 
     if (buf_size < 0x280) {
         av_log(ctx->avctx, AV_LOG_ERROR,
@@ -163,7 +170,8 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
         return AVERROR_INVALIDDATA;
     }
 
-    if (memcmp(buf, header_prefix, 5) && memcmp(buf, header_prefix444, 5) && memcmp(buf, header_prefixhr, 5)) {
+    if (memcmp(buf, header_prefix, 5) && memcmp(buf, header_prefix444, 5) &&
+        memcmp(buf, header_prefixhr1, 5) && memcmp(buf, header_prefixhr2, 5)) {
         av_log(ctx->avctx, AV_LOG_ERROR,
                "unknown header 0x%02X 0x%02X 0x%02X 0x%02X 0x%02X\n",
                buf[0], buf[1], buf[2], buf[3], buf[4]);
@@ -186,6 +194,7 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
     switch(buf[0x21] >> 5) {
     case 1: ctx->bit_depth = 8; break;
     case 2: ctx->bit_depth = 10; break;
+    case 3: ctx->bit_depth = 12; break;
     default:
         av_log(ctx->avctx, AV_LOG_ERROR,
                "Unknown bitdepth indicator (%d)\n", buf[0x21] >> 5);
@@ -198,9 +207,16 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
         if (ctx->bit_depth == 8) {
             avpriv_request_sample(ctx->avctx, "4:4:4 8 bits\n");
             return AVERROR_INVALIDDATA;
+        } else if (ctx->bit_depth == 10) {
+            ctx->decode_dct_block = dnxhd_decode_dct_block_10_444;
+            ctx->pix_fmt = AV_PIX_FMT_YUV444P10;
+        } else {
+            ctx->decode_dct_block = dnxhd_decode_dct_block_12_444;
+            ctx->pix_fmt = AV_PIX_FMT_YUV444P12;
         }
-        ctx->decode_dct_block = dnxhd_decode_dct_block_10_444;
-        ctx->pix_fmt = AV_PIX_FMT_YUV444P10;
+    } else if (ctx->bit_depth == 12) {
+        ctx->decode_dct_block = dnxhd_decode_dct_block_12;
+        ctx->pix_fmt = AV_PIX_FMT_YUV422P12;
     } else if (ctx->bit_depth == 10) {
         ctx->decode_dct_block = dnxhd_decode_dct_block_10;
         ctx->pix_fmt = AV_PIX_FMT_YUV422P10;
@@ -254,20 +270,40 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
            ctx->width, ctx->height, ctx->is_444 ? "4:4" : "2:2",
            ctx->bit_depth, ctx->mbaff, ctx->act);
 
-    if (ctx->mb_height > 68 ||
-        (ctx->mb_height << frame->interlaced_frame) > (ctx->height + 15) >> 4) {
+    // Newer format supports variable mb_scan_index sizes
+    if (!memcmp(buf, header_prefixhr2, 5)) {
+        ctx->data_offset = 0x170 + (ctx->mb_height << 2);
+    } else {
+        if (ctx->mb_height > 68 ||
+            (ctx->mb_height << frame->interlaced_frame) > (ctx->height + 15) >> 4) {
+            av_log(ctx->avctx, AV_LOG_ERROR,
+                   "mb height too big: %d\n", ctx->mb_height);
+            return AVERROR_INVALIDDATA;
+        }
+        ctx->data_offset = 0x280;
+    }
+
+    if (buf_size < ctx->data_offset) {
         av_log(ctx->avctx, AV_LOG_ERROR,
-               "mb height too big: %d\n", ctx->mb_height);
+               "buffer too small (%d < %d).\n", buf_size, ctx->data_offset);
         return AVERROR_INVALIDDATA;
     }
 
+    if (ctx->mb_height != old_mb_height) {
+        av_freep(&ctx->mb_scan_index);
+
+        ctx->mb_scan_index = av_mallocz_array(ctx->mb_height, sizeof(uint32_t));
+        if (!ctx->mb_scan_index)
+            return AVERROR(ENOMEM);
+    }
+
     for (i = 0; i < ctx->mb_height; i++) {
         ctx->mb_scan_index[i] = AV_RB32(buf + 0x170 + (i << 2));
-        ff_dlog(ctx->avctx, "mb scan index %d\n", ctx->mb_scan_index[i]);
-        if (buf_size < ctx->mb_scan_index[i] + 0x280LL) {
+        ff_dlog(ctx->avctx, "mb scan index %d, pos %d: %u\n", i, 0x170 + (i << 2), ctx->mb_scan_index[i]);
+        if (buf_size - ctx->data_offset < ctx->mb_scan_index[i]) {
             av_log(ctx->avctx, AV_LOG_ERROR,
-                   "invalid mb scan index (%d < %d).\n",
-                   buf_size, ctx->mb_scan_index[i] + 0x280);
+                   "invalid mb scan index (%u vs %u).\n",
+                   ctx->mb_scan_index[i], buf_size - ctx->data_offset);
             return AVERROR_INVALIDDATA;
         }
     }
@@ -280,7 +316,8 @@ static av_always_inline int dnxhd_decode_dct_block(const DNXHDContext *ctx,
                                                    int n,
                                                    int index_bits,
                                                    int level_bias,
-                                                   int level_shift)
+                                                   int level_shift,
+                                                   int dc_shift)
 {
     int i, j, index1, index2, len, flags;
     int level, component, sign;
@@ -323,7 +360,7 @@ static av_always_inline int dnxhd_decode_dct_block(const DNXHDContext *ctx,
         LAST_SKIP_BITS(bs, &row->gb, len);
         sign  = ~level >> 31;
         level = (NEG_USR32(sign ^ level, len) ^ sign) - sign;
-        row->last_dc[component] += level;
+        row->last_dc[component] += level << dc_shift;
     }
     block[0] = row->last_dc[component];
 
@@ -379,25 +416,37 @@ static av_always_inline int dnxhd_decode_dct_block(const DNXHDContext *ctx,
 static int dnxhd_decode_dct_block_8(const DNXHDContext *ctx,
                                     RowContext *row, int n)
 {
-    return dnxhd_decode_dct_block(ctx, row, n, 4, 32, 6);
+    return dnxhd_decode_dct_block(ctx, row, n, 4, 32, 6, 0);
 }
 
 static int dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
                                      RowContext *row, int n)
 {
-    return dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4);
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4, 0);
 }
 
 static int dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
                                          RowContext *row, int n)
 {
-    return dnxhd_decode_dct_block(ctx, row, n, 6, 32, 6);
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 32, 6, 0);
+}
+
+static int dnxhd_decode_dct_block_12(const DNXHDContext *ctx,
+                                     RowContext *row, int n)
+{
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4, 2);
+}
+
+static int dnxhd_decode_dct_block_12_444(const DNXHDContext *ctx,
+                                         RowContext *row, int n)
+{
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 32, 4, 2);
 }
 
 static int dnxhd_decode_macroblock(const DNXHDContext *ctx, RowContext *row,
                                    AVFrame *frame, int x, int y)
 {
-    int shift1 = ctx->bit_depth == 10;
+    int shift1 = ctx->bit_depth >= 10;
     int dct_linesize_luma   = frame->linesize[0];
     int dct_linesize_chroma = frame->linesize[1];
     uint8_t *dest_y, *dest_u, *dest_v;
@@ -556,8 +605,8 @@ decode_coding_unit:
         picture->key_frame = 1;
     }
 
-    ctx->buf_size = buf_size - 0x280;
-    ctx->buf = buf + 0x280;
+    ctx->buf_size = buf_size - ctx->data_offset;
+    ctx->buf = buf + ctx->data_offset;
     avctx->execute2(avctx, dnxhd_decode_row, picture, NULL, ctx->mb_height);
 
     if (first_field && picture->interlaced_frame) {
@@ -590,6 +639,7 @@ static av_cold int dnxhd_decode_close(AVCodecContext *avctx)
     ff_free_vlc(&ctx->dc_vlc);
     ff_free_vlc(&ctx->run_vlc);
 
+    av_freep(&ctx->mb_scan_index);
     av_freep(&ctx->rows);
 
     return 0;