dnxhdenc: fix invalid reads in dnxhd_mb_var_thread().

Do not assume that frame dimensions are mod16 (or that height is mod32
for interlaced).

CC:libav-stable@libav.org
(cherry picked from commit 69c25c9284645cf5189af2ede42d6f53828f3b45)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

1 files changed, 24 insertions, 3 deletions

diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c
index 8531fe0ea5..97e0fede2f 100644
--- a/libavcodec/dnxhdenc.c
+++ b/libavcodec/dnxhdenc.c
@@ -615,14 +615,35 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx)
 static int dnxhd_mb_var_thread(AVCodecContext *avctx, void *arg, int jobnr, int threadnr)
 {
     DNXHDEncContext *ctx = avctx->priv_data;
-    int mb_y = jobnr, mb_x;
+    int mb_y = jobnr, mb_x, x, y;
+    int partial_last_row = (mb_y == ctx->m.mb_height - 1) &&
+                           ((avctx->height >> ctx->interlaced) & 0xF);
+
     ctx = ctx->thread[threadnr];
     if (ctx->cid_table->bit_depth == 8) {
         uint8_t *pix = ctx->thread[0]->src[0] + ((mb_y<<4) * ctx->m.linesize);
         for (mb_x = 0; mb_x < ctx->m.mb_width; ++mb_x, pix += 16) {
             unsigned mb  = mb_y * ctx->m.mb_width + mb_x;
-            int sum = ctx->m.dsp.pix_sum(pix, ctx->m.linesize);
-            int varc = (ctx->m.dsp.pix_norm1(pix, ctx->m.linesize) - (((unsigned)sum*sum)>>8)+128)>>8;
+            int sum;
+            int varc;
+
+            if (!partial_last_row && mb_x * 16 <= avctx->width - 16) {
+                sum  = ctx->m.dsp.pix_sum(pix, ctx->m.linesize);
+                varc = ctx->m.dsp.pix_norm1(pix, ctx->m.linesize);
+            } else {
+                int bw = FFMIN(avctx->width - 16 * mb_x, 16);
+                int bh = FFMIN((avctx->height >> ctx->interlaced) - 16 * mb_y, 16);
+                sum = varc = 0;
+                for (y = 0; y < bh; y++) {
+                    for (x = 0; x < bw; x++) {
+                        uint8_t val = pix[x + y * ctx->m.linesize];
+                        sum  += val;
+                        varc += val * val;
+                    }
+                }
+            }
+            varc = (varc - (((unsigned)sum * sum) >> 8) + 128) >> 8;
+
             ctx->mb_cmp[mb].value = varc;
             ctx->mb_cmp[mb].mb = mb;
         }