openh264enc: Remove broken byte-stream to avc conversion and just output byte-stream as generated by the encoder

The byte-stream to avc conversion did not consider NAL sizes bigger than 2^16,
multiple layers, multiple NALs per layer, and various other things. This
caused corrupted streams in higher bitrates and other circumstances.

Let's just forward byte-stream as generated by the encoder and let h264parse
handle conversion to avc if needed. That way we only have to keep around one
version of the conversion and don't have to fix it in multiple places.

1 files changed, 31 insertions, 114 deletions

diff --git a/ext/openh264/gstopenh264enc.cpp b/ext/openh264/gstopenh264enc.cpp
index 6d5be721e..295153792 100644
--- a/ext/openh264/gstopenh264enc.cpp
+++ b/ext/openh264/gstopenh264enc.cpp
@@ -214,7 +214,7 @@ GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS
-    ("video/x-h264, stream-format=(string)\"avc\", alignment=(string)\"au\", profile=(string)\"baseline\"")
+    ("video/x-h264, stream-format=(string)\"byte-stream\", alignment=(string)\"au\", profile=(string)\"baseline\"")
     );
 
 /* class initialization */
@@ -473,8 +473,7 @@ gst_openh264enc_set_property (GObject * object, guint property_id,
       break;
 
     case PROP_COMPLEXITY:
-      openh264enc->complexity =
-          (ECOMPLEXITY_MODE) g_value_get_enum (value);
+      openh264enc->complexity = (ECOMPLEXITY_MODE) g_value_get_enum (value);
       break;
 
     default:
@@ -615,17 +614,9 @@ gst_openh264enc_set_format (GstVideoEncoder * encoder,
 {
   GstOpenh264Enc *openh264enc = GST_OPENH264ENC (encoder);
   gchar *debug_caps;
-  SFrameBSInfo bsInfo;
   guint width, height, fps_n, fps_d;
   SEncParamExt enc_params;
   gint ret;
-  guchar *nal_sps_data = NULL;
-  gint nal_sps_length = 0;
-  guchar *nal_pps_data = NULL;
-  gint nal_pps_length = 0;
-  guchar *sps_tmp_buf;
-  guchar *codec_data_tmp_buf;
-  GstBuffer *codec_data;
   GstCaps *outcaps;
   GstVideoCodecState *output_state;
   openh264enc->frame_count = 0;
@@ -671,11 +662,9 @@ gst_openh264enc_set_format (GstVideoEncoder * encoder,
   enc_params.bEnableDenoise = openh264enc->enable_denoise;
   enc_params.iComplexityMode = openh264enc->complexity;
   enc_params.uiIntraPeriod = openh264enc->gop_size;
-  enc_params.bEnableBackgroundDetection =
-      openh264enc->background_detection;
+  enc_params.bEnableBackgroundDetection = openh264enc->background_detection;
   enc_params.bEnableAdaptiveQuant = openh264enc->adaptive_quantization;
-  enc_params.bEnableSceneChangeDetect =
-      openh264enc->scene_change_detection;
+  enc_params.bEnableSceneChangeDetect = openh264enc->scene_change_detection;
   enc_params.bEnableFrameSkip = openh264enc->enable_frame_skip;
   enc_params.bEnableLongTermReference = 0;
 #if OPENH264_MINOR >= 4
@@ -691,8 +680,7 @@ gst_openh264enc_set_format (GstVideoEncoder * encoder,
   enc_params.sSpatialLayers[0].iVideoHeight = height;
   enc_params.sSpatialLayers[0].fFrameRate = fps_n * 1.0 / fps_d;
   enc_params.sSpatialLayers[0].iSpatialBitrate = openh264enc->bitrate;
-  enc_params.sSpatialLayers[0].sSliceCfg.uiSliceMode =
-      openh264enc->slice_mode;
+  enc_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = openh264enc->slice_mode;
   enc_params.sSpatialLayers[0].sSliceCfg.sSliceArgument.uiSliceNum =
       openh264enc->num_slices;
 
@@ -707,56 +695,9 @@ gst_openh264enc_set_format (GstVideoEncoder * encoder,
 
   openh264enc->encoder->SetOption (ENCODER_OPTION_DATAFORMAT, &video_format);
 
-  memset (&bsInfo, 0, sizeof (SFrameBSInfo));
-
-  ret = openh264enc->encoder->EncodeParameterSets (&bsInfo);
-
-  nal_sps_data = bsInfo.sLayerInfo[0].pBsBuf + 4;
-  nal_sps_length = bsInfo.sLayerInfo[0].pNalLengthInByte[0] - 4;
-
-  nal_pps_data = bsInfo.sLayerInfo[0].pBsBuf + nal_sps_length + 8;
-  nal_pps_length = bsInfo.sLayerInfo[0].pNalLengthInByte[1] - 4;
-
-  if (ret != cmResultSuccess) {
-    GST_ELEMENT_ERROR (openh264enc, STREAM, ENCODE,
-        ("Could not create headers"), ("Could not create SPS"));
-    return FALSE;
-  }
-
-  sps_tmp_buf = (guchar *) (g_memdup (nal_sps_data, nal_sps_length));
-
-  codec_data_tmp_buf =
-      (guchar *) g_malloc (5 + 3 + nal_sps_length + 3 + nal_pps_length);
-  codec_data_tmp_buf[0] = 1; /* version 1 */ ;
-  codec_data_tmp_buf[1] = sps_tmp_buf[1];       /* profile */
-  codec_data_tmp_buf[2] = sps_tmp_buf[2];       /* profile constraints */
-  codec_data_tmp_buf[3] = sps_tmp_buf[3];       /* level */
-  codec_data_tmp_buf[4] = 1;    /* NAL length marker length minus one */
-  codec_data_tmp_buf[5] = 1;    /* Number of SPS */
-  GST_WRITE_UINT16_BE (codec_data_tmp_buf + 6, nal_sps_length);
-  memcpy (codec_data_tmp_buf + 8, sps_tmp_buf, nal_sps_length);
-
-  g_free (sps_tmp_buf);
-
-  codec_data_tmp_buf[8 + nal_sps_length] = 1;   /* Number of PPS */
-  GST_WRITE_UINT16_BE (codec_data_tmp_buf + 8 + nal_sps_length + 1,
-      nal_pps_length);
-  memcpy (codec_data_tmp_buf + 8 + nal_sps_length + 3, nal_pps_data,
-      nal_pps_length);
-
-  GST_DEBUG_OBJECT (openh264enc, "Got SPS of size %d and PPS of size %d",
-      nal_sps_length, nal_pps_length);
-
-  codec_data =
-      gst_buffer_new_wrapped (codec_data_tmp_buf,
-      5 + 3 + nal_sps_length + 3 + nal_pps_length);
-
   outcaps =
       gst_caps_copy (gst_static_pad_template_get_caps
       (&gst_openh264enc_src_template));
-  gst_caps_set_simple (outcaps, "codec_data", GST_TYPE_BUFFER, codec_data,
-      NULL);
-  gst_buffer_unref (codec_data);
 
   output_state = gst_video_encoder_set_output_state (encoder, outcaps, state);
   gst_video_codec_state_unref (output_state);
@@ -785,7 +726,9 @@ gst_openh264enc_handle_frame (GstVideoEncoder * encoder,
   gint ret;
   SFrameBSInfo frame_info;
   gfloat fps;
-  GstVideoEncoder *base_encoder = GST_VIDEO_ENCODER (openh264enc);
+  GstMapInfo map;
+  gint i, j;
+  gsize buf_length = 0;
 
   if (frame) {
     src_pic = new SSourcePicture;
@@ -803,8 +746,7 @@ gst_openh264enc_handle_frame (GstVideoEncoder * encoder,
   openh264enc->frame_count++;
   if (frame) {
     if (G_UNLIKELY (openh264enc->frame_count == 1)) {
-      openh264enc->time_per_frame =
-          (GST_NSECOND / openh264enc->framerate);
+      openh264enc->time_per_frame = (GST_NSECOND / openh264enc->framerate);
       openh264enc->previous_timestamp = frame->pts;
     } else {
       openh264enc->time_per_frame =
@@ -865,7 +807,7 @@ gst_openh264enc_handle_frame (GstVideoEncoder * encoder,
   if (videoFrameTypeSkip == frame_info.eFrameType) {
     if (frame) {
       gst_video_frame_unmap (&video_frame);
-      gst_video_encoder_finish_frame (base_encoder, frame);
+      gst_video_encoder_finish_frame (encoder, frame);
       delete src_pic;
     }
 
@@ -883,7 +825,7 @@ gst_openh264enc_handle_frame (GstVideoEncoder * encoder,
   /* FIXME: openh264 has no way for us to get a connection
    * between the input and output frames, we just have to
    * guess based on the input */
-  frame = gst_video_encoder_get_oldest_frame (base_encoder);
+  frame = gst_video_encoder_get_oldest_frame (encoder);
   if (!frame) {
     GST_ELEMENT_ERROR (openh264enc, STREAM, ENCODE,
         ("Could not encode frame"), ("openh264enc returned %d", ret));
@@ -891,59 +833,34 @@ gst_openh264enc_handle_frame (GstVideoEncoder * encoder,
     return GST_FLOW_ERROR;
   }
 
-  SLayerBSInfo *bs_info = &frame_info.sLayerInfo[0];
-  gint nal_size = bs_info->pNalLengthInByte[0] - 4;
-  guchar *nal_sps_data, *nal_pps_data;
-  gint nal_sps_length, nal_pps_length, idr_length, tmp_buf_length;
-
   if (videoFrameTypeIDR == frame_info.eFrameType) {
-    GstMapInfo map;
-
-    /* sps */
-    nal_sps_data = frame_info.sLayerInfo[0].pBsBuf + 4;
-    nal_sps_length = frame_info.sLayerInfo[0].pNalLengthInByte[0] - 4;
-    /* pps */
-    nal_pps_data = nal_sps_data + frame_info.sLayerInfo[0].pNalLengthInByte[0];
-    nal_pps_length = frame_info.sLayerInfo[0].pNalLengthInByte[1] - 4;
-    /* idr */
-    bs_info = &frame_info.sLayerInfo[1];
-    idr_length = bs_info->pNalLengthInByte[0] - 4;
-
-    tmp_buf_length = nal_sps_length + 2 + nal_pps_length + 2 + idr_length + 2;
-    frame->output_buffer =
-        gst_video_encoder_allocate_output_buffer (encoder, tmp_buf_length);
-    gst_buffer_map (frame->output_buffer, &map, GST_MAP_WRITE);
-
-    GST_WRITE_UINT16_BE (map.data, nal_sps_length);
-    memcpy (map.data + 2, nal_sps_data, nal_sps_length);
-
-    GST_WRITE_UINT16_BE (map.data + nal_sps_length + 2, nal_pps_length);
-    memcpy (map.data + nal_sps_length + 2 + 2, nal_pps_data, nal_pps_length);
-
-    GST_WRITE_UINT16_BE (map.data + nal_sps_length + 2 + nal_pps_length + 2,
-        idr_length);
-    memcpy (map.data + nal_sps_length + 2 + nal_pps_length + 2 + 2,
-        bs_info->pBsBuf + 4, idr_length);
-
-    gst_buffer_unmap (frame->output_buffer, &map);
-
     GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame);
   } else {
-    GstMapInfo map;
-
-    tmp_buf_length = nal_size + 2;
-    frame->output_buffer =
-        gst_video_encoder_allocate_output_buffer (encoder, tmp_buf_length);
-    gst_buffer_map (frame->output_buffer, &map, GST_MAP_WRITE);
+    GST_VIDEO_CODEC_FRAME_UNSET_SYNC_POINT (frame);
+  }
 
-    GST_WRITE_UINT16_BE (map.data, nal_size);
-    memcpy (map.data + 2, bs_info->pBsBuf + 4, nal_size);
+  for (i = 0; i < frame_info.iLayerNum; i++) {
+    for (j = 0; j < frame_info.sLayerInfo[i].iNalCount; j++) {
+      buf_length += frame_info.sLayerInfo[i].pNalLengthInByte[j];
+    }
+  }
 
-    gst_buffer_unmap (frame->output_buffer, &map);
+  frame->output_buffer =
+      gst_video_encoder_allocate_output_buffer (encoder, buf_length);
+  gst_buffer_map (frame->output_buffer, &map, GST_MAP_WRITE);
 
-    GST_VIDEO_CODEC_FRAME_UNSET_SYNC_POINT (frame);
+  buf_length = 0;
+  for (i = 0; i < frame_info.iLayerNum; i++) {
+    gsize layer_size = 0;
+    for (j = 0; j < frame_info.sLayerInfo[i].iNalCount; j++) {
+      layer_size += frame_info.sLayerInfo[i].pNalLengthInByte[j];
+    }
+    memcpy (map.data + buf_length, frame_info.sLayerInfo[i].pBsBuf, layer_size);
+    buf_length += layer_size;
   }
 
+  gst_buffer_unmap (frame->output_buffer, &map);
+
   GST_LOG_OBJECT (openh264enc, "openh264 picture %scoded OK!",
       (ret != cmResultSuccess) ? "NOT " : "");