encoder: fix computation of max coded buffer size.

Fix coded buffer size for each codec. A generic issue was that the
number of macroblocks was incorrectly computed. The second issue was
specific to MPEG-2 were the max number of bits per macroblock, and
as defined by the standard, was incorrectly mapped to the (lower)
H.264 requirement. i.e. 4608 bits vs. 3200 bits limit.

3 files changed, 11 insertions, 3 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.c b/gst-libs/gst/vaapi/gstvaapiencoder.c
index e56886fa..e0be3141 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.c
@@ -371,9 +371,6 @@ gst_vaapi_encoder_set_format (GstVaapiEncoder * encoder,
   if (!gst_vaapi_encoder_ensure_context (encoder))
     goto error;
 
-  encoder->codedbuf_size = (GST_VAAPI_ENCODER_WIDTH (encoder) *
-      GST_VAAPI_ENCODER_HEIGHT (encoder) * 400) / (16 * 16);
-
   encoder->codedbuf_pool = gst_vaapi_coded_buffer_pool_new (encoder,
       encoder->codedbuf_size);
   if (!encoder->codedbuf_pool) {
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index d23646f2..a9a9b136 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -1594,6 +1594,12 @@ gst_vaapi_encoder_h264_set_context_info (GstVaapiEncoder * base_encoder)
 
   cip->profile = encoder->profile;
   cip->ref_frames = (encoder->b_frame_num ? 2 : 1) + DEFAULT_SURFACES_COUNT;
+
+  /* Only YUV 4:2:0 formats are supported for now. This means that we
+     have a limit of 3200 bits per macroblock. */
+  /* XXX: check profile and compute RawMbBits */
+  base_encoder->codedbuf_size = (GST_ROUND_UP_16 (cip->width) *
+      GST_ROUND_UP_16 (cip->height) / 256) * 400;
 }
 
 static GstCaps *
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
index 876dc6d9..90a6e23d 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_mpeg2.c
@@ -635,6 +635,11 @@ gst_vaapi_encoder_mpeg2_set_context_info (GstVaapiEncoder * base_encoder)
 
   cip->profile = to_vaapi_profile (encoder->profile);
   cip->ref_frames = 2;
+
+  /* Only YUV 4:2:0 formats are supported for now. This means that we
+     have a limit of 4608 bits per macroblock. */
+  base_encoder->codedbuf_size = (GST_ROUND_UP_16 (cip->width) *
+      GST_ROUND_UP_16 (cip->height) / 256) * 576;
 }
 
 static gboolean