avcodec/mss2: calculate draw region and revise split position

for videos with wmv9 rectangles, the region drawn by ff_mss12_decode_rect
may be less than the entire video area. the wmv9 rectangles are used to
calculate the ff_mss12_decode_rect draw region.

Fixes tickets #3255 and #4043

1 files changed, 66 insertions, 4 deletions

diff --git a/libavcodec/mss2.c b/libavcodec/mss2.c
index d8a30019f7..69494d8c44 100644
--- a/libavcodec/mss2.c
+++ b/libavcodec/mss2.c
@@ -468,6 +468,39 @@ struct Rectangle {
     int coded, x, y, w, h;
 };
 
+struct Rectangle2 {
+    int left, right, top, bottom;
+};
+
+static void calc_draw_region(struct Rectangle2 * draw, const struct Rectangle2 * rect)
+{
+#define COMPARE(top, bottom, left, right)  \
+    if (rect->top <= draw->top && rect->bottom >= draw->bottom) { \
+        if (rect->left <= draw->left && rect->right >= draw->left) \
+            draw->left = FFMIN(rect->right, draw->right); \
+        \
+        if (rect->right >= draw->right) { \
+            if (rect->left >= draw->left) { \
+                if (rect->left < draw->right) \
+                    draw->right = rect->left; \
+            } else { \
+                draw->right = draw->left; \
+            } \
+        } \
+    }
+
+    COMPARE(top, bottom, left, right)
+    COMPARE(left, right, top, bottom)
+}
+
+static int calc_split_position(int split_position, const struct Rectangle2 * rect, int height)
+{
+    if (rect->top || rect->bottom != height)
+        split_position = rect->top + split_position * (rect->bottom - rect->top) / height;
+
+    return av_clip(split_position, rect->top + 1, rect->bottom - 1);
+}
+
 #define MAX_WMV9_RECTANGLES 20
 #define ARITH2_PADDING 2
 
@@ -485,6 +518,7 @@ static int mss2_decode_frame(AVCodecContext *avctx, AVFrame *frame,
     int keyframe, has_wmv9, has_mv, is_rle, is_555, ret;
 
     struct Rectangle wmv9rects[MAX_WMV9_RECTANGLES], *r;
+    struct Rectangle2 draw;
     int used_rects = 0, i, implicit_rect = 0, av_uninit(wmv9_mask);
 
     if ((ret = init_get_bits8(&gb, buf, buf_size)) < 0)
@@ -671,11 +705,32 @@ static int mss2_decode_frame(AVCodecContext *avctx, AVFrame *frame,
             bytestream2_init(&gB, buf, buf_size + ARITH2_PADDING);
             arith2_init(&acoder, &gB);
             c->keyframe = keyframe;
-            if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[0], &acoder, 0, 0,
-                                                    avctx->width,
-                                                    ctx->split_position))
+
+            draw.left = 0;
+            draw.top = 0;
+            draw.right = avctx->width;
+            draw.bottom = avctx->height;
+            if (wmv9_mask == -1) {
+                for (i = 0; i < used_rects; i++) {
+                    struct Rectangle2 r;
+                    r.left   = wmv9rects[i].x;
+                    r.top    = wmv9rects[i].y;
+                    r.right  = r.left + wmv9rects[i].w;
+                    r.bottom = r.top + wmv9rects[i].h;
+                    calc_draw_region(&draw, &r);
+                }
+            }
+
+            if (draw.left >= avctx->width || draw.right > avctx->width ||
+                draw.top >= avctx->height || draw.bottom > avctx->height)
                 return AVERROR_INVALIDDATA;
 
+            if (c->slice_split && draw.bottom - draw.top >= 10) {
+                ctx->split_position = calc_split_position(ctx->split_position, &draw, avctx->height);
+            if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[0], &acoder, 0, draw.top,
+                                                    avctx->width,
+                                                    ctx->split_position - draw.top))
+                return AVERROR_INVALIDDATA;
             buf      += arith2_get_consumed_bytes(&acoder);
             buf_size -= arith2_get_consumed_bytes(&acoder);
             if (c->slice_split) {
@@ -686,7 +741,14 @@ static int mss2_decode_frame(AVCodecContext *avctx, AVFrame *frame,
                 if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[1], &acoder, 0,
                                                         ctx->split_position,
                                                         avctx->width,
-                                                        avctx->height - ctx->split_position))
+                                                        draw.bottom - ctx->split_position))
+                    return AVERROR_INVALIDDATA;
+                buf      += arith2_get_consumed_bytes(&acoder);
+                buf_size -= arith2_get_consumed_bytes(&acoder);
+            }
+            } else {
+                if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[0], &acoder, draw.left, draw.top,
+                                                        draw.right - draw.left, draw.bottom - draw.top))
                     return AVERROR_INVALIDDATA;
 
                 buf      += arith2_get_consumed_bytes(&acoder);