omxvideoenc: Add long-term-ref support to ZYNQ_USCALE_PLUS encoder

Custom API that upstream elements can use to notify encoders about
marking longterm ref. pictures or using longterm ref. pictures in
encoding process.

This patch adds below properties:

long-term-ref: Enable/Disable dynamically marking long-term
reference pictures in encoding process

long-term-freq: Periodicity of long-term reference picture
marking in encoding process.

If a picture is marked as long-term reference picture then it remains
in the DPB list for ever unless it overrides with new long-term pitcure with
same index. Encoder can use this long-term picture as refence for
encoding.

This feature is mostly useful to avoid visual artifacts propagation in streaming use cases
when packet loss happens. Instead of requesting for IDR, client can request for use long-term
reference picture for encoding.

2 files changed, 126 insertions, 0 deletions

diff --git a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
index c98d239..f3f2454 100644
--- a/omx/gstomxvideoenc.c
+++ b/omx/gstomxvideoenc.c
@@ -253,6 +253,9 @@ static GstFlowReturn gst_omx_video_enc_drain (GstOMXVideoEnc * self);
 static GstFlowReturn gst_omx_video_enc_handle_output_frame (GstOMXVideoEnc *
     self, GstOMXPort * port, GstOMXBuffer * buf, GstVideoCodecFrame * frame);
 
+static gboolean gst_omx_video_enc_sink_event (GstVideoEncoder * encoder,
+    GstEvent * event);
+
 enum
 {
   PROP_0,
@@ -277,6 +280,8 @@ enum
   PROP_SLICE_SIZE,
   PROP_DEPENDENT_SLICE,
   PROP_DEFAULT_ROI_QUALITY,
+  PROP_LONGTERM_REF,
+  PROP_LONGTERM_FREQUENCY,
 };
 
 /* FIXME: Better defaults */
@@ -301,6 +306,12 @@ enum
 #define GST_OMX_VIDEO_ENC_SLICE_SIZE_DEFAULT (0)
 #define GST_OMX_VIDEO_ENC_DEPENDENT_SLICE_DEFAULT (FALSE)
 #define GST_OMX_VIDEO_ENC_DEFAULT_ROI_QUALITY OMX_ALG_ROI_QUALITY_HIGH
+#define GST_OMX_VIDEO_ENC_LONGTERM_REF_DEFAULT (FALSE)
+#define GST_OMX_VIDEO_ENC_LONGTERM_FREQUENCY_DEFAULT (0)
+
+/* ZYNQ_USCALE_PLUS encoder custom events */
+#define OMX_ALG_GST_EVENT_INSERT_LONGTERM "omx-alg/insert-longterm"
+#define OMX_ALG_GST_EVENT_USE_LONGTERM "omx-alg/use-longterm"
 
 /* class initialization */
 #define do_init \
@@ -484,6 +495,22 @@ gst_omx_video_enc_class_init (GstOMXVideoEncClass * klass)
           GST_TYPE_OMX_VIDEO_ENC_ROI_QUALITY,
           GST_OMX_VIDEO_ENC_DEFAULT_ROI_QUALITY,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_LONGTERM_REF,
+      g_param_spec_boolean ("long-term-ref", "LongTerm Reference Pictures",
+          "If enabled, encoder accepts dynamically inserting and using long-term reference "
+          "picture events from upstream elements",
+          GST_OMX_VIDEO_ENC_LONGTERM_REF_DEFAULT,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
+          GST_PARAM_MUTABLE_READY));
+
+  g_object_class_install_property (gobject_class, PROP_LONGTERM_FREQUENCY,
+      g_param_spec_uint ("long-term-freq", "LongTerm reference frequency",
+          "Periodicity of LongTerm reference picture marking in encoding process "
+          "Units in frames, distance between two consequtive long-term reference pictures",
+          0, G_MAXUINT, GST_OMX_VIDEO_ENC_LONGTERM_REF_DEFAULT,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
+          GST_PARAM_MUTABLE_READY));
 #endif
 
   element_class->change_state =
@@ -502,6 +529,8 @@ gst_omx_video_enc_class_init (GstOMXVideoEncClass * klass)
   video_encoder_class->propose_allocation =
       GST_DEBUG_FUNCPTR (gst_omx_video_enc_propose_allocation);
   video_encoder_class->getcaps = GST_DEBUG_FUNCPTR (gst_omx_video_enc_getcaps);
+  video_encoder_class->sink_event =
+      GST_DEBUG_FUNCPTR (gst_omx_video_enc_sink_event);
   video_encoder_class->decide_allocation =
       GST_DEBUG_FUNCPTR (gst_omx_video_enc_decide_allocation);
 
@@ -538,6 +567,8 @@ gst_omx_video_enc_init (GstOMXVideoEnc * self)
   self->slice_size = GST_OMX_VIDEO_ENC_SLICE_SIZE_DEFAULT;
   self->dependent_slice = GST_OMX_VIDEO_ENC_DEPENDENT_SLICE_DEFAULT;
   self->default_roi_quality = GST_OMX_VIDEO_ENC_DEFAULT_ROI_QUALITY;
+  self->long_term_ref = GST_OMX_VIDEO_ENC_LONGTERM_REF_DEFAULT;
+  self->long_term_freq = GST_OMX_VIDEO_ENC_LONGTERM_FREQUENCY_DEFAULT;
 #endif
 
   self->default_target_bitrate = GST_OMX_PROP_OMX_DEFAULT;
@@ -808,6 +839,22 @@ set_zynqultrascaleplus_props (GstOMXVideoEnc * self)
     CHECK_ERR ("slices");
   }
 
+  {
+    OMX_ALG_VIDEO_PARAM_LONG_TERM longterm;
+    GST_OMX_INIT_STRUCT (&longterm);
+    longterm.nPortIndex = self->enc_out_port->index;
+    longterm.bEnableLongTerm = self->long_term_ref;
+    longterm.nLongTermFrequency = self->long_term_freq;
+
+    GST_DEBUG_OBJECT (self, "setting long-term ref to %d, long-term-freq to %d",
+        self->long_term_ref, self->long_term_freq);
+
+    err =
+        gst_omx_component_set_parameter (self->enc,
+        (OMX_INDEXTYPE) OMX_ALG_IndexParamVideoLongTerm, &longterm);
+    CHECK_ERR ("longterm");
+  }
+
   return TRUE;
 }
 #endif
@@ -1140,6 +1187,12 @@ gst_omx_video_enc_set_property (GObject * object, guint prop_id,
     case PROP_DEFAULT_ROI_QUALITY:
       self->default_roi_quality = g_value_get_enum (value);
       break;
+    case PROP_LONGTERM_REF:
+      self->long_term_ref = g_value_get_boolean (value);
+      break;
+    case PROP_LONGTERM_FREQUENCY:
+      self->long_term_freq = g_value_get_uint (value);
+      break;
 #endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@@ -1220,6 +1273,12 @@ gst_omx_video_enc_get_property (GObject * object, guint prop_id, GValue * value,
     case PROP_DEFAULT_ROI_QUALITY:
       g_value_set_enum (value, self->default_roi_quality);
       break;
+    case PROP_LONGTERM_REF:
+      g_value_set_boolean (value, self->long_term_ref);
+      break;
+    case PROP_LONGTERM_FREQUENCY:
+      g_value_set_uint (value, self->long_term_freq);
+      break;
 #endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@@ -3366,6 +3425,71 @@ gst_omx_video_enc_getcaps (GstVideoEncoder * encoder, GstCaps * filter)
   return ret;
 }
 
+#ifdef USE_OMX_TARGET_ZYNQ_USCALE_PLUS
+static gboolean
+handle_longterm_event (GstOMXVideoEnc * self, GstEvent * event)
+{
+  OMX_ALG_VIDEO_CONFIG_INSERT longterm;
+  OMX_ERRORTYPE err;
+  OMX_INDEXTYPE omx_index_long_term;
+
+  GST_OMX_INIT_STRUCT (&longterm);
+  longterm.nPortIndex = self->enc_in_port->index;
+
+  /* If long-term-ref is enabled then "omx-alg/insert-longterm" event
+   * marks the encoding picture as long term reference picture and
+   * "omx-alg/use-longterm" event informs the encoder that encoding picture
+   * should use existing long term picture in the dpb as reference for encoding process */
+
+  if (self->long_term_ref) {
+    if (gst_event_has_name (event, OMX_ALG_GST_EVENT_INSERT_LONGTERM)) {
+      GST_LOG_OBJECT (self, "received omx-alg/insert-longterm event");
+      omx_index_long_term =
+          (OMX_INDEXTYPE) OMX_ALG_IndexConfigVideoInsertLongTerm;
+    } else {
+      GST_LOG_OBJECT (self, "received omx-alg/use-longterm event");
+      omx_index_long_term = (OMX_INDEXTYPE) OMX_ALG_IndexConfigVideoUseLongTerm;
+    }
+
+    err =
+        gst_omx_component_set_config (self->enc, omx_index_long_term,
+        &longterm);
+
+    if (err != OMX_ErrorNone)
+      GST_ERROR_OBJECT (self,
+          "Failed to longterm events: %s (0x%08x)",
+          gst_omx_error_to_string (err), err);
+  } else {
+    GST_WARNING_OBJECT (self,
+        "LongTerm events are not handled because long_term_ref is disabled");
+  }
+
+  return TRUE;
+}
+#endif
+
+static gboolean
+gst_omx_video_enc_sink_event (GstVideoEncoder * encoder, GstEvent * event)
+{
+  switch (GST_EVENT_TYPE (event)) {
+    case GST_EVENT_CUSTOM_DOWNSTREAM:
+    {
+#ifdef USE_OMX_TARGET_ZYNQ_USCALE_PLUS
+      GstOMXVideoEnc *self = GST_OMX_VIDEO_ENC (encoder);
+      if (gst_event_has_name (event, OMX_ALG_GST_EVENT_INSERT_LONGTERM)
+          || gst_event_has_name (event, OMX_ALG_GST_EVENT_USE_LONGTERM))
+        return handle_longterm_event (self, event);
+#endif
+    }
+    default:
+      break;
+  }
+
+  return
+      GST_VIDEO_ENCODER_CLASS (gst_omx_video_enc_parent_class)->sink_event
+      (encoder, event);
+}
+
 static gboolean
 gst_omx_video_enc_decide_allocation (GstVideoEncoder * encoder,
     GstQuery * query)
diff --git a/omx/gstomxvideoenc.h b/omx/gstomxvideoenc.h
index c84e62a..80b1473 100644
--- a/omx/gstomxvideoenc.h
+++ b/omx/gstomxvideoenc.h
@@ -92,6 +92,8 @@ struct _GstOMXVideoEnc
   guint32 slice_size;
   gboolean dependent_slice;
   gint default_roi_quality;
+  gboolean long_term_ref;
+  guint32 long_term_freq;
 #endif
 
   guint32 default_target_bitrate;