libavcodec: v4l2: add support for v4l2 mem2mem codecs

    This patchset enhances Alexis Ballier's original patch and validates
    it using Qualcomm's Venus hardware (driver recently landed upstream
    [1]).

    This has been tested on Qualcomm's DragonBoard 410c and 820c
    Configure/make scripts have been validated on Ubuntu 10.04 and
    16.04.

    Tested decoders:
           - h264
           - h263
           - mpeg4
           - vp8
           - vp9
           - hevc

    Tested encoders:
           - h264
           - h263
           - mpeg4

    Tested transcoding (concurrent encoding/decoding)

    Some of the changes introduced:
        - v4l2: code cleanup and abstractions added
        - v4l2: follow the new encode/decode api.
        - v4l2: fix display size for NV12 output pool.
        - v4l2: handle EOS (EPIPE and draining)
        - v4l2: vp8 and mpeg4 decoding and encoding.
        - v4l2: hevc and vp9 support.
        - v4l2: generate EOF on dequeue errors.
        - v4l2: h264_mp4toannexb filtering.
        - v4l2: fixed make install and fate issues.
        - v4l2: codecs enabled/disabled depending on pixfmt defined
        - v4l2: pass timebase/framerate to the context
        - v4l2: runtime decoder reconfiguration.
        - v4l2: add more frame information
        - v4l2: free hardware resources on last reference being released
        - v4l2: encoding: disable b-frames for upstreaming (patch required)

    [1] https://lwn.net/Articles/697956/

    System Level view:
        v42l_m2m_enc/dec --> v4l2_m2m --> v4l2_context --> v4l2_buffers

    Reviewed-by: Jorge Ramirez <jorge.ramirez-ortiz@linaro.org>
    Reviewed-by: Alexis Ballier <aballier@gentoo.org>
    Tested-by: Jorge Ramirez <jorge.ramirez-ortiz@linaro.org>

Signed-off-by: wm4 <nfxjfg@googlemail.com>

1 files changed, 383 insertions, 0 deletions

diff --git a/libavcodec/v4l2_m2m.c b/libavcodec/v4l2_m2m.c
new file mode 100644
index 0000000000..1d7a8521d8
--- /dev/null
+++ b/libavcodec/v4l2_m2m.c
@@ -0,0 +1,383 @@
+/*
+ * V4L mem2mem
+ *
+ * Copyright (C) 2017 Alexis Ballier <aballier@gentoo.org>
+ * Copyright (C) 2017 Jorge Ramirez <jorge.ramirez-ortiz@linaro.org>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <linux/videodev2.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include "libavcodec/avcodec.h"
+#include "libavcodec/internal.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/pixfmt.h"
+#include "v4l2_context.h"
+#include "v4l2_fmt.h"
+#include "v4l2_m2m.h"
+
+static inline int v4l2_splane_video(struct v4l2_capability *cap)
+{
+    if (cap->capabilities & (V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT) &&
+        cap->capabilities & V4L2_CAP_STREAMING)
+        return 1;
+
+    if (cap->capabilities & V4L2_CAP_VIDEO_M2M)
+        return 1;
+
+    return 0;
+}
+
+static inline int v4l2_mplane_video(struct v4l2_capability *cap)
+{
+    if (cap->capabilities & (V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_OUTPUT_MPLANE) &&
+        cap->capabilities & V4L2_CAP_STREAMING)
+        return 1;
+
+    if (cap->capabilities & V4L2_CAP_VIDEO_M2M_MPLANE)
+        return 1;
+
+    return 0;
+}
+
+static int v4l2_prepare_contexts(V4L2m2mContext* s)
+{
+    struct v4l2_capability cap;
+    int ret;
+
+    s->capture.done = s->output.done = 0;
+    s->capture.name = "capture";
+    s->output.name = "output ";
+    atomic_init(&s->refcount, 0);
+    sem_init(&s->refsync, 0, 0);
+
+    memset(&cap, 0, sizeof(cap));
+    ret = ioctl(s->fd, VIDIOC_QUERYCAP, &cap);
+    if (ret < 0)
+        return ret;
+
+    av_log(s->avctx, AV_LOG_INFO, "driver '%s' on card '%s'\n", cap.driver, cap.card);
+
+    if (v4l2_mplane_video(&cap)) {
+        s->capture.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+        s->output.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+        return 0;
+    }
+
+    if (v4l2_splane_video(&cap)) {
+        s->capture.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+        s->output.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+        return 0;
+    }
+
+    return AVERROR(EINVAL);
+}
+
+static int v4l2_probe_driver(V4L2m2mContext* s)
+{
+    int ret;
+
+    s->fd = open(s->devname, O_RDWR | O_NONBLOCK, 0);
+    if (s->fd < 0)
+        return AVERROR(errno);
+
+    ret = v4l2_prepare_contexts(s);
+    if (ret < 0)
+        goto done;
+
+    ret = ff_v4l2_context_get_format(&s->output);
+    if (ret) {
+        av_log(s->avctx, AV_LOG_DEBUG, "v4l2 output format not supported\n");
+        goto done;
+    }
+
+    ret = ff_v4l2_context_get_format(&s->capture);
+    if (ret) {
+        av_log(s->avctx, AV_LOG_DEBUG, "v4l2 capture format not supported\n");
+        goto done;
+    }
+
+done:
+    if (close(s->fd) < 0) {
+        ret = AVERROR(errno);
+        av_log(s->avctx, AV_LOG_ERROR, "failure closing %s (%s)\n", s->devname, av_err2str(AVERROR(errno)));
+    }
+
+    s->fd = -1;
+
+    return ret;
+}
+
+static int v4l2_configure_contexts(V4L2m2mContext* s)
+{
+    void *log_ctx = s->avctx;
+    int ret;
+
+    s->fd = open(s->devname, O_RDWR | O_NONBLOCK, 0);
+    if (s->fd < 0)
+        return AVERROR(errno);
+
+    ret = v4l2_prepare_contexts(s);
+    if (ret < 0)
+        goto error;
+
+    ret = ff_v4l2_context_set_format(&s->output);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "can't set v4l2 output format\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_set_format(&s->capture);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "can't to set v4l2 capture format\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_init(&s->output);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "no v4l2 output context's buffers\n");
+        goto error;
+    }
+
+    /* decoder's buffers need to be updated at a later stage */
+    if (!av_codec_is_decoder(s->avctx->codec)) {
+        ret = ff_v4l2_context_init(&s->capture);
+        if (ret) {
+            av_log(log_ctx, AV_LOG_ERROR, "no v4l2 capture context's buffers\n");
+            goto error;
+        }
+    }
+
+    return 0;
+
+error:
+    if (close(s->fd) < 0) {
+        ret = AVERROR(errno);
+        av_log(log_ctx, AV_LOG_ERROR, "error closing %s (%s)\n",
+            s->devname, av_err2str(AVERROR(errno)));
+    }
+    s->fd = -1;
+
+    return ret;
+}
+
+/******************************************************************************
+ *
+ *                  V4L2 M2M Interface
+ *
+ ******************************************************************************/
+int ff_v4l2_m2m_codec_reinit(V4L2m2mContext* s)
+{
+    int ret;
+
+    av_log(s->avctx, AV_LOG_DEBUG, "reinit context\n");
+
+    /* 1. streamoff */
+    ret = ff_v4l2_context_set_status(&s->capture, VIDIOC_STREAMOFF);
+    if (ret)
+        av_log(s->avctx, AV_LOG_ERROR, "capture VIDIOC_STREAMOFF\n");
+
+    /* 2. unmap the capture buffers (v4l2 and ffmpeg):
+     *    we must wait for all references to be released before being allowed
+     *    to queue new buffers.
+     */
+    av_log(s->avctx, AV_LOG_DEBUG, "waiting for user to release AVBufferRefs\n");
+    if (atomic_load(&s->refcount))
+        while(sem_wait(&s->refsync) == -1 && errno == EINTR);
+
+    ff_v4l2_context_release(&s->capture);
+
+    /* 3. get the new capture format */
+    ret = ff_v4l2_context_get_format(&s->capture);
+    if (ret) {
+        av_log(s->avctx, AV_LOG_ERROR, "query the new capture format\n");
+        return ret;
+    }
+
+    /* 4. set the capture format */
+    ret = ff_v4l2_context_set_format(&s->capture);
+    if (ret) {
+        av_log(s->avctx, AV_LOG_ERROR, "setting capture format\n");
+        return ret;
+    }
+
+    /* 5. complete reinit */
+    sem_destroy(&s->refsync);
+    sem_init(&s->refsync, 0, 0);
+    s->draining = 0;
+    s->reinit = 0;
+
+    return 0;
+}
+
+int ff_v4l2_m2m_codec_full_reinit(V4L2m2mContext *s)
+{
+    void *log_ctx = s->avctx;
+    int ret;
+
+    av_log(log_ctx, AV_LOG_DEBUG, "%s full reinit\n", s->devname);
+
+    /* wait for pending buffer references */
+    if (atomic_load(&s->refcount))
+        while(sem_wait(&s->refsync) == -1 && errno == EINTR);
+
+    /* close the driver */
+    ff_v4l2_m2m_codec_end(s->avctx);
+
+    /* start again now that we know the stream dimensions */
+    s->draining = 0;
+    s->reinit = 0;
+
+    s->fd = open(s->devname, O_RDWR | O_NONBLOCK, 0);
+    if (s->fd < 0)
+        return AVERROR(errno);
+
+    ret = v4l2_prepare_contexts(s);
+    if (ret < 0)
+        goto error;
+
+    /* if a full re-init was requested - probe didn't run - we need to populate
+     * the format for each context
+     */
+    ret = ff_v4l2_context_get_format(&s->output);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_DEBUG, "v4l2 output format not supported\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_get_format(&s->capture);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_DEBUG, "v4l2 capture format not supported\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_set_format(&s->output);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "can't set v4l2 output format\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_set_format(&s->capture);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "can't to set v4l2 capture format\n");
+        goto error;
+    }
+
+    ret = ff_v4l2_context_init(&s->output);
+    if (ret) {
+        av_log(log_ctx, AV_LOG_ERROR, "no v4l2 output context's buffers\n");
+        goto error;
+    }
+
+    /* decoder's buffers need to be updated at a later stage */
+    if (!av_codec_is_decoder(s->avctx->codec)) {
+        ret = ff_v4l2_context_init(&s->capture);
+        if (ret) {
+            av_log(log_ctx, AV_LOG_ERROR, "no v4l2 capture context's buffers\n");
+            goto error;
+        }
+    }
+
+    return 0;
+
+error:
+    if (close(s->fd) < 0) {
+        ret = AVERROR(errno);
+        av_log(log_ctx, AV_LOG_ERROR, "error closing %s (%s)\n",
+            s->devname, av_err2str(AVERROR(errno)));
+    }
+    s->fd = -1;
+
+    return ret;
+}
+
+int ff_v4l2_m2m_codec_end(AVCodecContext *avctx)
+{
+    V4L2m2mContext* s = avctx->priv_data;
+    int ret;
+
+    ret = ff_v4l2_context_set_status(&s->output, VIDIOC_STREAMOFF);
+    if (ret)
+            av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->output.name);
+
+    ret = ff_v4l2_context_set_status(&s->capture, VIDIOC_STREAMOFF);
+    if (ret)
+        av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->capture.name);
+
+    ff_v4l2_context_release(&s->output);
+
+    if (atomic_load(&s->refcount))
+        av_log(avctx, AV_LOG_ERROR, "ff_v4l2m2m_codec_end leaving pending buffers\n");
+
+    ff_v4l2_context_release(&s->capture);
+    sem_destroy(&s->refsync);
+
+    /* release the hardware */
+    if (close(s->fd) < 0 )
+        av_log(avctx, AV_LOG_ERROR, "failure closing %s (%s)\n", s->devname, av_err2str(AVERROR(errno)));
+
+    s->fd = -1;
+
+    return 0;
+}
+
+int ff_v4l2_m2m_codec_init(AVCodecContext *avctx)
+{
+    int ret = AVERROR(EINVAL);
+    struct dirent *entry;
+    char node[PATH_MAX];
+    DIR *dirp;
+
+    V4L2m2mContext *s = avctx->priv_data;
+    s->avctx = avctx;
+
+    dirp = opendir("/dev");
+    if (!dirp)
+        return AVERROR(errno);
+
+    for (entry = readdir(dirp); entry; entry = readdir(dirp)) {
+
+        if (strncmp(entry->d_name, "video", 5))
+            continue;
+
+        snprintf(node, sizeof(node), "/dev/%s", entry->d_name);
+        av_log(s->avctx, AV_LOG_DEBUG, "probing device %s\n", node);
+        strncpy(s->devname, node, strlen(node) + 1);
+        ret = v4l2_probe_driver(s);
+        if (!ret)
+                break;
+    }
+
+    closedir(dirp);
+
+    if (ret) {
+        av_log(s->avctx, AV_LOG_ERROR, "Could not find a valid device\n");
+        memset(s->devname, 0, sizeof(s->devname));
+
+        return ret;
+    }
+
+    av_log(s->avctx, AV_LOG_INFO, "Using device %s\n", node);
+
+    return v4l2_configure_contexts(s);
+}