encoder: add rate control API.

Add gst_vaapi_encoder_set_rate_control() interface to request a new
rate control mode for encoding. Changing the rate control mode is
only valid prior to encoding the very first frame. Afterwards, an
error ("operation-failed") is issued.

https://bugzilla.gnome.org/show_bug.cgi?id=719529

7 files changed, 178 insertions, 3 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.c b/gst-libs/gst/vaapi/gstvaapiencoder.c
index a9cd85d8..05f9b3d2 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.c
@@ -25,6 +25,7 @@
 #include "gstvaapiencoder_priv.h"
 #include "gstvaapicontext.h"
 #include "gstvaapidisplay_priv.h"
+#include "gstvaapiutils.h"
 
 #define DEBUG 1
 #include "gstvaapidebug.h"
@@ -180,6 +181,7 @@ gst_vaapi_encoder_put_frame (GstVaapiEncoder * encoder,
     gst_vaapi_coded_buffer_proxy_set_user_data (codedbuf_proxy,
         picture, (GDestroyNotify) gst_vaapi_enc_picture_unref);
     g_async_queue_push (encoder->codedbuf_queue, codedbuf_proxy);
+    encoder->num_codedbuf_queued++;
 
     /* Try again with any pending reordered frame now available for encoding */
     frame = NULL;
@@ -388,6 +390,107 @@ error:
   return NULL;
 }
 
+/* Determine the supported rate control modes */
+static gboolean
+get_rate_control_mask (GstVaapiEncoder * encoder)
+{
+  const GstVaapiEncoderClassData *const cdata =
+      GST_VAAPI_ENCODER_GET_CLASS (encoder)->class_data;
+  GstVaapiProfile profile;
+  GArray *profiles;
+  guint i, rate_control_mask = 0;
+
+  if (encoder->rate_control_mask)
+    return encoder->rate_control_mask;
+
+  profiles = gst_vaapi_display_get_encode_profiles (encoder->display);
+  if (!profiles)
+    goto cleanup;
+
+  // Pick a profile matching the class codec
+  for (i = 0; i < profiles->len; i++) {
+    profile = g_array_index (profiles, GstVaapiProfile, i);
+    if (gst_vaapi_profile_get_codec (profile) == cdata->codec)
+      break;
+  }
+
+  if (i != profiles->len) {
+    VAConfigAttrib attrib;
+    VAStatus status;
+
+    attrib.type = VAConfigAttribRateControl;
+    GST_VAAPI_DISPLAY_LOCK (encoder->display);
+    status =
+        vaGetConfigAttributes (GST_VAAPI_DISPLAY_VADISPLAY (encoder->display),
+        gst_vaapi_profile_get_va_profile (profile), VAEntrypointEncSlice,
+        &attrib, 1);
+    GST_VAAPI_DISPLAY_UNLOCK (encoder->display);
+    if (vaapi_check_status (status, "vaGetConfigAttributes()")) {
+      for (i = 0; i < 32; i++) {
+        if (!(attrib.value & (1 << i)))
+          continue;
+        rate_control_mask |= 1 << to_GstVaapiRateControl (1 << i);
+      }
+    }
+  }
+  g_array_unref (profiles);
+  GST_INFO ("supported rate controls: 0x%08x", rate_control_mask);
+
+cleanup:
+  encoder->rate_control_mask = cdata->rate_control_mask & rate_control_mask;
+  return encoder->rate_control_mask;
+}
+
+/**
+ * gst_vaapi_encoder_set_rate_control:
+ * @encoder: a #GstVaapiEncoder
+ * @rate_control: the requested rate control
+ *
+ * Notifies the @encoder to use the supplied @rate_control mode.
+ *
+ * If the underlying encoder does not support that rate control mode,
+ * then @GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_RATE_CONTROL is
+ * returned.
+ *
+ * The rate control mode can only be specified before the first frame
+ * is to be encoded. Afterwards, any change to this parameter is
+ * invalid and @GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED is
+ * returned.
+ *
+ * Return value: a #GstVaapiEncoderStatus
+ */
+GstVaapiEncoderStatus
+gst_vaapi_encoder_set_rate_control (GstVaapiEncoder * encoder,
+    GstVaapiRateControl rate_control)
+{
+  guint32 rate_control_mask;
+
+  g_return_val_if_fail (encoder != NULL,
+      GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_PARAMETER);
+
+  if (encoder->rate_control != rate_control && encoder->num_codedbuf_queued > 0)
+    goto error_operation_failed;
+
+  rate_control_mask = get_rate_control_mask (encoder);
+  if (!(rate_control_mask & (1U << rate_control)))
+    goto error_unsupported_rate_control;
+
+  encoder->rate_control = rate_control;
+  return GST_VAAPI_ENCODER_STATUS_SUCCESS;
+
+  /* ERRORS */
+error_operation_failed:
+  {
+    GST_ERROR ("could not change rate control mode after encoding started");
+    return GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED;
+  }
+error_unsupported_rate_control:
+  {
+    GST_ERROR ("unsupported rate control mode (%d)", rate_control);
+    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_RATE_CONTROL;
+  }
+}
+
 /* Base encoder initialization (internal) */
 static gboolean
 gst_vaapi_encoder_init (GstVaapiEncoder * encoder, GstVaapiDisplay * display)
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.h b/gst-libs/gst/vaapi/gstvaapiencoder.h
index d548b5b1..03305414 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.h
@@ -40,6 +40,11 @@ typedef struct _GstVaapiEncoder GstVaapiEncoder;
  *   hold the encoded picture.
  * @GST_VAAPI_ENCODER_STATUS_ERROR_UNKNOWN: Unknown error.
  * @GST_VAAPI_ENCODER_STATUS_ERROR_ALLOCATION_FAILED: No memory left.
+ * @GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED: The requested
+ *   operation failed to execute properly. e.g. invalid point in time to
+ *   execute the operation.
+ * @GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_RATE_CONTROL:
+ *   Unsupported rate control value.
  * @GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_PARAMETER: Invalid parameter.
  * @GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_BUFFER: Invalid buffer.
  * @GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_SURFACE: Invalid surface.
@@ -55,6 +60,8 @@ typedef enum
 
   GST_VAAPI_ENCODER_STATUS_ERROR_UNKNOWN = -1,
   GST_VAAPI_ENCODER_STATUS_ERROR_ALLOCATION_FAILED = -2,
+  GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED = -3,
+  GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_RATE_CONTROL = -4,
   GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_PARAMETER = -100,
   GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_BUFFER = -101,
   GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_SURFACE = -102,
@@ -80,6 +87,10 @@ gst_vaapi_encoder_set_format (GstVaapiEncoder * encoder,
     GstVideoCodecState * state, GstCaps * ref_caps);
 
 GstVaapiEncoderStatus
+gst_vaapi_encoder_set_rate_control (GstVaapiEncoder * encoder,
+    GstVaapiRateControl rate_control);
+
+GstVaapiEncoderStatus
 gst_vaapi_encoder_put_frame (GstVaapiEncoder * encoder,
     GstVideoCodecFrame * frame);
 
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index d5bef53c..2da7f5d4 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -36,6 +36,17 @@
 #define DEBUG 1
 #include "gstvaapidebug.h"
 
+/* Define default rate control mode ("constant-qp") */
+#define DEFAULT_RATECONTROL GST_VAAPI_RATECONTROL_CQP
+
+/* Supported set of VA rate controls, within this implementation */
+#define SUPPORTED_RATECONTROLS                          \
+  (GST_VAAPI_RATECONTROL_MASK (NONE) |                  \
+   GST_VAAPI_RATECONTROL_MASK (CQP)  |                  \
+   GST_VAAPI_RATECONTROL_MASK (CBR)  |                  \
+   GST_VAAPI_RATECONTROL_MASK (VBR)  |                  \
+   GST_VAAPI_RATECONTROL_MASK (VBR_CONSTRAINED))
+
 #define GST_VAAPI_ENCODER_H264_NAL_REF_IDC_NONE        0
 #define GST_VAAPI_ENCODER_H264_NAL_REF_IDC_LOW         1
 #define GST_VAAPI_ENCODER_H264_NAL_REF_IDC_MEDIUM      2
@@ -1690,6 +1701,8 @@ gst_vaapi_encoder_h264_init (GstVaapiEncoder * base)
 {
   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264 (base);
 
+  base->rate_control = DEFAULT_RATECONTROL;
+
   /* init attributes */
   encoder->profile = 0;
   encoder->level = 0;
@@ -1756,6 +1769,8 @@ gst_vaapi_encoder_h264_finalize (GstVaapiEncoder * base)
 
 }
 
+GST_VAAPI_ENCODER_DEFINE_CLASS_DATA (H264);
+
 static inline const GstVaapiEncoderClass *
 gst_vaapi_encoder_h264_class (void)
 {
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
index c4b6b201..3c1c34b6 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
@@ -37,6 +37,14 @@
 #define DEBUG 1
 #include "gstvaapidebug.h"
 
+/* Define default rate control mode ("constant-qp") */
+#define DEFAULT_RATECONTROL GST_VAAPI_RATECONTROL_CQP
+
+/* Supported set of VA rate controls, within this implementation */
+#define SUPPORTED_RATECONTROLS                  \
+  (GST_VAAPI_RATECONTROL_MASK (NONE) |          \
+   GST_VAAPI_RATECONTROL_MASK (CQP)  |          \
+   GST_VAAPI_RATECONTROL_MASK (CBR))
 
 static gboolean
 gst_bit_writer_write_sps (GstBitWriter * bitwriter,
@@ -720,6 +728,8 @@ gst_vaapi_encoder_mpeg2_init (GstVaapiEncoder * base)
 {
   GstVaapiEncoderMpeg2 *encoder = GST_VAAPI_ENCODER_MPEG2 (base);
 
+  base->rate_control = DEFAULT_RATECONTROL;
+
   /* re-ordering */
   g_queue_init (&encoder->b_frames);
   encoder->dump_frames = FALSE;
@@ -778,6 +788,8 @@ gst_vaapi_encoder_mpeg2_finalize (GstVaapiEncoder * base)
   g_queue_clear (&encoder->b_frames);
 }
 
+GST_VAAPI_ENCODER_DEFINE_CLASS_DATA (MPEG2);
+
 static inline const GstVaapiEncoderClass *
 gst_vaapi_encoder_mpeg2_class (void)
 {
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2_priv.h b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2_priv.h
index ff20dbe5..71e14041 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2_priv.h
@@ -48,7 +48,6 @@ typedef enum
 
 #define GST_VAAPI_ENCODER_MPEG2_DEFAULT_PROFILE      GST_ENCODER_MPEG2_PROFILE_MAIN
 #define GST_VAAPI_ENCODER_MPEG2_DEFAULT_LEVEL        GST_VAAPI_ENCODER_MPEG2_LEVEL_HIGH
-#define GST_VAAPI_ENCODER_MPEG2_DEFAULT_RATE_CONTROL GST_VAAPI_RATECONTROL_CQP
 
 #define GST_VAAPI_ENCODER_MPEG2_MIN_CQP                 2
 #define GST_VAAPI_ENCODER_MPEG2_MAX_CQP                 62
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_priv.h b/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
index bd17c8f4..a3ec6db5 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
@@ -61,8 +61,17 @@ G_BEGIN_DECLS
 #define GST_VAAPI_ENCODER_HEIGHT(encoder)     (GST_VAAPI_ENCODER_CAST(encoder)->video_info.height)
 #define GST_VAAPI_ENCODER_FPS_N(encoder)      (GST_VAAPI_ENCODER_CAST(encoder)->video_info.fps_n)
 #define GST_VAAPI_ENCODER_FPS_D(encoder)      (GST_VAAPI_ENCODER_CAST(encoder)->video_info.fps_d)
-#define GST_VAAPI_ENCODER_RATE_CONTROL(encoder)   \
-    (GST_VAAPI_ENCODER_CAST(encoder)->rate_control)
+
+/**
+ * GST_VAAPI_ENCODER_RATE_CONTROL:
+ * @encoder: a #GstVaapiEncoder
+ *
+ * Macro that evaluates to the rate control.
+ * This is an internal macro that does not do any run-time type check.
+ */
+#undef  GST_VAAPI_ENCODER_RATE_CONTROL
+#define GST_VAAPI_ENCODER_RATE_CONTROL(encoder) \
+  (GST_VAAPI_ENCODER_CAST (encoder)->rate_control)
 
 #define GST_VAAPI_ENCODER_CHECK_STATUS(exp, err_num, err_reason, ...)   \
   if (!(exp)) {                                                         \
@@ -72,6 +81,7 @@ G_BEGIN_DECLS
   }
 
 typedef struct _GstVaapiEncoderClass GstVaapiEncoderClass;
+typedef struct _GstVaapiEncoderClassData GstVaapiEncoderClassData;
 
 struct _GstVaapiEncoder
 {
@@ -87,6 +97,7 @@ struct _GstVaapiEncoder
   VAContextID va_context;
   GstVideoInfo video_info;
   GstVaapiRateControl rate_control;
+  guint32 rate_control_mask;
 
   GMutex mutex;
   GCond surface_free;
@@ -94,13 +105,32 @@ struct _GstVaapiEncoder
   guint codedbuf_size;
   GstVaapiVideoPool *codedbuf_pool;
   GAsyncQueue *codedbuf_queue;
+  guint32 num_codedbuf_queued;
 };
 
+struct _GstVaapiEncoderClassData
+{
+  /*< private >*/
+  GstVaapiCodec codec;
+
+  GstVaapiRateControl default_rate_control;
+  guint32 rate_control_mask;
+};
+
+#define GST_VAAPI_ENCODER_DEFINE_CLASS_DATA(CODEC)                      \
+  static const GstVaapiEncoderClassData g_class_data = {                \
+    .codec = G_PASTE (GST_VAAPI_CODEC_, CODEC),                         \
+    .default_rate_control = DEFAULT_RATECONTROL,                        \
+    .rate_control_mask = SUPPORTED_RATECONTROLS,                        \
+  }
+
 struct _GstVaapiEncoderClass
 {
   /*< private >*/
   GstVaapiMiniObjectClass parent_class;
 
+  const GstVaapiEncoderClassData *class_data;
+
   gboolean              (*init)         (GstVaapiEncoder * encoder);
   void                  (*finalize)     (GstVaapiEncoder * encoder);
 
@@ -135,6 +165,7 @@ struct _GstVaapiEncoderClass
 
 #define GST_VAAPI_ENCODER_CLASS_INIT(CODEC, codec)              \
   GST_VAAPI_ENCODER_CLASS_INIT_BASE (CODEC),                    \
+    .class_data = &g_class_data,                                \
     GST_VAAPI_ENCODER_CLASS_HOOK (codec, init),                 \
     GST_VAAPI_ENCODER_CLASS_HOOK (codec, finalize),             \
     GST_VAAPI_ENCODER_CLASS_HOOK (codec, set_format),           \
diff --git a/gst-libs/gst/vaapi/gstvaapitypes.h b/gst-libs/gst/vaapi/gstvaapitypes.h
index 771ad75d..159b85b3 100644
--- a/gst-libs/gst/vaapi/gstvaapitypes.h
+++ b/gst-libs/gst/vaapi/gstvaapitypes.h
@@ -144,6 +144,10 @@ typedef enum {
     GST_VAAPI_RATECONTROL_VBR_CONSTRAINED,
 } GstVaapiRateControl;
 
+/* Define a mask for GstVaapiRateControl */
+#define GST_VAAPI_RATECONTROL_MASK(RC) \
+    (1 << G_PASTE(GST_VAAPI_RATECONTROL_,RC))
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_TYPES_H */