avviddec: Enable subframe decoding for H.264

Enable sending NAL units to the decoder without having to first
group them in a frame (an AU).

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-libav/-/merge_requests/66>

3 files changed, 46 insertions, 1 deletions

diff --git a/docs/gst_plugins_cache.json b/docs/gst_plugins_cache.json
index 71a80d5..dbe88b7 100644
--- a/docs/gst_plugins_cache.json
+++ b/docs/gst_plugins_cache.json
@@ -9436,7 +9436,7 @@
                 "long-name": "libav H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 decoder",
                 "pad-templates": {
                     "sink": {
-                        "caps": "video/x-h264:\n      alignment: au\n  stream-format: { (string)avc, (string)byte-stream }\n",
+                        "caps": "video/x-h264:\n      alignment: au\n  stream-format: { (string)avc, (string)byte-stream }\nvideo/x-h264:\n      alignment: nal\n  stream-format: byte-stream\n",
                         "direction": "sink",
                         "presence": "always"
                     },
diff --git a/ext/libav/gstavcodecmap.c b/ext/libav/gstavcodecmap.c
index d0421c9..3b775e4 100644
--- a/ext/libav/gstavcodecmap.c
+++ b/ext/libav/gstavcodecmap.c
@@ -1291,6 +1291,11 @@ gst_ffmpeg_codecid_to_caps (enum AVCodecID codec_id,
         g_value_unset (&item);
         gst_caps_set_value (caps, "stream-format", &arr);
         g_value_unset (&arr);
+
+        gst_caps_append (caps, gst_ff_vid_caps_new (context, NULL, codec_id,
+                encode, "video/x-h264", "alignment", G_TYPE_STRING, "nal",
+                "stream-format", G_TYPE_STRING, "byte-stream", NULL));
+
       } else if (context) {
         /* FIXME: ffmpeg currently assumes AVC if there is extradata and
          * byte-stream otherwise. See for example the MOV or MPEG-TS code.
diff --git a/ext/libav/gstavviddec.c b/ext/libav/gstavviddec.c
index e35025a..a14b6df 100644
--- a/ext/libav/gstavviddec.c
+++ b/ext/libav/gstavviddec.c
@@ -362,6 +362,18 @@ gst_ffmpegviddec_context_set_flags (AVCodecContext * context, guint flags,
     context->flags &= ~flags;
 }
 
+static void
+gst_ffmpegviddec_context_set_flags2 (AVCodecContext * context, guint flags,
+    gboolean enable)
+{
+  g_return_if_fail (context != NULL);
+
+  if (enable)
+    context->flags2 |= flags;
+  else
+    context->flags2 &= ~flags;
+}
+
 /* with LOCK */
 static gboolean
 gst_ffmpegviddec_close (GstFFMpegVidDec * ffmpegdec, gboolean reset)
@@ -575,6 +587,24 @@ gst_ffmpegviddec_set_format (GstVideoDecoder * decoder,
   } else
     ffmpegdec->context->thread_count = ffmpegdec->max_threads;
 
+  if (oclass->in_plugin->id == AV_CODEC_ID_H264) {
+    GstStructure *s = gst_caps_get_structure (state->caps, 0);
+    const char *alignment;
+    gboolean nal_aligned;
+
+    alignment = gst_structure_get_string (s, "alignment");
+    nal_aligned = !g_strcmp0 (alignment, "nal");
+    if (nal_aligned) {
+      if (ffmpegdec->context->thread_type == FF_THREAD_FRAME)
+        goto nal_only_slice;
+      ffmpegdec->context->thread_type = FF_THREAD_SLICE;
+    }
+
+    gst_ffmpegviddec_context_set_flags2 (ffmpegdec->context,
+        AV_CODEC_FLAG2_CHUNKS, nal_aligned);
+    gst_video_decoder_set_subframe_mode (decoder, nal_aligned);
+  }
+
   /* open codec - we don't select an output pix_fmt yet,
    * simply because we don't know! We only get it
    * during playback... */
@@ -614,6 +644,12 @@ open_failed:
     GST_DEBUG_OBJECT (ffmpegdec, "Failed to open");
     goto done;
   }
+nal_only_slice:
+  {
+    GST_ERROR_OBJECT (ffmpegdec,
+        "Can't do NAL aligned H.264 with frame threading.");
+    goto done;
+  }
 }
 
 typedef struct
@@ -1826,6 +1862,10 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec,
     GST_VIDEO_CODEC_FRAME_FLAG_UNSET (frame,
         GST_FFMPEG_VIDEO_CODEC_FRAME_FLAG_ALLOCATED);
 
+  if (gst_video_decoder_get_subframe_mode (GST_VIDEO_DECODER (ffmpegdec)))
+    gst_video_decoder_have_last_subframe (GST_VIDEO_DECODER (ffmpegdec),
+        out_frame);
+
   /* FIXME: Ideally we would remap the buffer read-only now before pushing but
    * libav might still have a reference to it!
    */