avcodec/cuvid: add support for cropping/resizing

Overhauled version, original patch by Miroslav Slugeň <thunder.m@email.cz>.

1 files changed, 66 insertions, 19 deletions

diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
index c0b4a37734..916d7e9987 100644
--- a/libavcodec/cuvid.c
+++ b/libavcodec/cuvid.c
@@ -43,6 +43,20 @@ typedef struct CuvidContext
     char *cu_gpu;
     int nb_surfaces;
     int drop_second_field;
+    char *crop_expr;
+    char *resize_expr;
+
+    struct {
+        int left;
+        int top;
+        int right;
+        int bottom;
+    } crop;
+
+    struct {
+        int width;
+        int height;
+    } resize;
 
     AVBufferRef *hwdevice;
     AVBufferRef *hwframe;
@@ -107,17 +121,46 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
     CUVIDDECODECREATEINFO cuinfo;
     int surface_fmt;
 
+    int old_width = avctx->width;
+    int old_height = avctx->height;
+
     enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
                                        AV_PIX_FMT_NONE,  // Will be updated below
                                        AV_PIX_FMT_NONE };
 
     av_log(avctx, AV_LOG_TRACE, "pfnSequenceCallback, progressive_sequence=%d\n", format->progressive_sequence);
 
+    memset(&cuinfo, 0, sizeof(cuinfo));
+
     ctx->internal_error = 0;
 
+    avctx->coded_width = cuinfo.ulWidth = format->coded_width;
+    avctx->coded_height = cuinfo.ulHeight = format->coded_height;
+
+    // apply cropping
+    cuinfo.display_area.left = format->display_area.left + ctx->crop.left;
+    cuinfo.display_area.top = format->display_area.top + ctx->crop.top;
+    cuinfo.display_area.right = format->display_area.right - ctx->crop.right;
+    cuinfo.display_area.bottom = format->display_area.bottom - ctx->crop.bottom;
+
     // width and height need to be set before calling ff_get_format
-    avctx->width = format->display_area.right;
-    avctx->height = format->display_area.bottom;
+    if (ctx->resize_expr) {
+        avctx->width = ctx->resize.width;
+        avctx->height = ctx->resize.height;
+    } else {
+        avctx->width = cuinfo.display_area.right - cuinfo.display_area.left;
+        avctx->height = cuinfo.display_area.bottom - cuinfo.display_area.top;
+    }
+
+    // target width/height need to be multiples of two
+    cuinfo.ulTargetWidth = avctx->width = (avctx->width + 1) & ~1;
+    cuinfo.ulTargetHeight = avctx->height = (avctx->height + 1) & ~1;
+
+    // aspect ratio conversion, 1:1, depends on scaled resolution
+    cuinfo.target_rect.left = 0;
+    cuinfo.target_rect.top = 0;
+    cuinfo.target_rect.right = cuinfo.ulTargetWidth;
+    cuinfo.target_rect.bottom = cuinfo.ulTargetHeight;
 
     switch (format->bit_depth_luma_minus8) {
     case 0: // 8-bit
@@ -195,6 +238,8 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
     if (ctx->cudecoder
             && avctx->coded_width == format->coded_width
             && avctx->coded_height == format->coded_height
+            && avctx->width == old_width
+            && avctx->height == old_height
             && ctx->chroma_format == format->chroma_format
             && ctx->codec_type == format->codec)
         return 1;
@@ -228,13 +273,8 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
         return 0;
     }
 
-    avctx->coded_width = format->coded_width;
-    avctx->coded_height = format->coded_height;
-
     ctx->chroma_format = format->chroma_format;
 
-    memset(&cuinfo, 0, sizeof(cuinfo));
-
     cuinfo.CodecType = ctx->codec_type = format->codec;
     cuinfo.ChromaFormat = format->chroma_format;
 
@@ -252,16 +292,6 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form
         return 0;
     }
 
-    cuinfo.ulWidth = avctx->coded_width;
-    cuinfo.ulHeight = avctx->coded_height;
-    cuinfo.ulTargetWidth = cuinfo.ulWidth;
-    cuinfo.ulTargetHeight = cuinfo.ulHeight;
-
-    cuinfo.target_rect.left = 0;
-    cuinfo.target_rect.top = 0;
-    cuinfo.target_rect.right = cuinfo.ulWidth;
-    cuinfo.target_rect.bottom = cuinfo.ulHeight;
-
     cuinfo.ulNumDecodeSurfaces = ctx->nb_surfaces;
     cuinfo.ulNumOutputSurfaces = 1;
     cuinfo.ulCreationFlags = cudaVideoCreate_PreferCUVID;
@@ -486,7 +516,7 @@ static int cuvid_output_frame(AVCodecContext *avctx, AVFrame *frame)
                 if (ret < 0)
                     goto error;
 
-                offset += avctx->coded_height;
+                offset += avctx->height;
             }
         } else if (avctx->pix_fmt == AV_PIX_FMT_NV12 ||
                    avctx->pix_fmt == AV_PIX_FMT_P010 ||
@@ -502,7 +532,7 @@ static int cuvid_output_frame(AVCodecContext *avctx, AVFrame *frame)
             tmp_frame->hw_frames_ctx = av_buffer_ref(ctx->hwframe);
             tmp_frame->data[0]       = (uint8_t*)mapped_frame;
             tmp_frame->linesize[0]   = pitch;
-            tmp_frame->data[1]       = (uint8_t*)(mapped_frame + avctx->coded_height * pitch);
+            tmp_frame->data[1]       = (uint8_t*)(mapped_frame + avctx->height * pitch);
             tmp_frame->linesize[1]   = pitch;
             tmp_frame->width         = avctx->width;
             tmp_frame->height        = avctx->height;
@@ -708,6 +738,21 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
     }
     avctx->pix_fmt = ret;
 
+    if (ctx->resize_expr && sscanf(ctx->resize_expr, "%dx%d",
+                                   &ctx->resize.width, &ctx->resize.height) != 2) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid resize expressions\n");
+        ret = AVERROR(EINVAL);
+        goto error;
+    }
+
+    if (ctx->crop_expr && sscanf(ctx->crop_expr, "%dx%dx%dx%d",
+                                 &ctx->crop.top, &ctx->crop.bottom,
+                                 &ctx->crop.left, &ctx->crop.right) != 4) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid cropping expressions\n");
+        ret = AVERROR(EINVAL);
+        goto error;
+    }
+
     ret = cuvid_load_functions(&ctx->cvdl);
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR, "Failed loading nvcuvid.\n");
@@ -953,6 +998,8 @@ static const AVOption options[] = {
     { "gpu",      "GPU to be used for decoding", OFFSET(cu_gpu), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, VD },
     { "surfaces", "Maximum surfaces to be used for decoding", OFFSET(nb_surfaces), AV_OPT_TYPE_INT, { .i64 = 25 }, 0, INT_MAX, VD },
     { "drop_second_field", "Drop second field when deinterlacing", OFFSET(drop_second_field), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VD },
+    { "crop",     "Crop (top)x(bottom)x(left)x(right)", OFFSET(crop_expr), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, VD },
+    { "resize",   "Resize (width)x(height)", OFFSET(resize_expr), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, VD },
     { NULL }
 };