h264parse: extract CEA-708 closed captions

Expose SEI data in the H.264 bitstream parser API and
extract closed captions and other things that are not
specified in the H.264 spec itself in the videoparser.

Based on patch by: Mathieu Duponchelle <mathieu@centricular.com>

https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/940

4 files changed, 185 insertions, 0 deletions

diff --git a/gst-libs/gst/codecparsers/gsth264parser.c b/gst-libs/gst/codecparsers/gsth264parser.c
index 30d6aea49..acc6fd91a 100644
--- a/gst-libs/gst/codecparsers/gsth264parser.c
+++ b/gst-libs/gst/codecparsers/gsth264parser.c
@@ -1009,6 +1009,51 @@ error:
 }
 
 static GstH264ParserResult
+gst_h264_parser_parse_registered_user_data (GstH264NalParser * nalparser,
+    GstH264RegisteredUserData * rud, NalReader * nr, guint payload_size)
+{
+  guint8 *data = NULL;
+  guint i;
+
+  rud->data = NULL;
+  rud->size = 0;
+
+  if (payload_size < 2)
+    return GST_H264_PARSER_ERROR;
+
+  READ_UINT8 (nr, rud->country_code, 8);
+  --payload_size;
+
+  if (rud->country_code == 0xFF) {
+    READ_UINT8 (nr, rud->country_code_extension, 8);
+    --payload_size;
+  } else {
+    rud->country_code_extension = 0;
+  }
+
+  if (payload_size < 8)
+    return GST_H264_PARSER_ERROR;
+
+  data = g_malloc (payload_size);
+  for (i = 0; i < payload_size / 8; ++i) {
+    READ_UINT8 (nr, data[i], 8);
+  }
+
+  GST_MEMDUMP ("SEI user data", data, payload_size / 8);
+
+  rud->data = data;
+  rud->size = payload_size;
+  return GST_H264_PARSER_OK;
+
+error:
+  {
+    GST_WARNING ("error parsing \"Registered User Data\"");
+    g_free (data);
+    return GST_H264_PARSER_ERROR;
+  }
+}
+
+static GstH264ParserResult
 gst_h264_parser_parse_recovery_point (GstH264NalParser * nalparser,
     GstH264RecoveryPoint * rp, NalReader * nr)
 {
@@ -1157,6 +1202,10 @@ gst_h264_parser_parse_sei_message (GstH264NalParser * nalparser,
       res = gst_h264_parser_parse_pic_timing (nalparser,
           &sei->payload.pic_timing, nr);
       break;
+    case GST_H264_SEI_REGISTERED_USER_DATA:
+      res = gst_h264_parser_parse_registered_user_data (nalparser,
+          &sei->payload.registered_user_data, nr, payload_size);
+      break;
     case GST_H264_SEI_RECOVERY_POINT:
       res = gst_h264_parser_parse_recovery_point (nalparser,
           &sei->payload.recovery_point, nr);
@@ -2276,6 +2325,22 @@ gst_h264_sps_clear (GstH264SPS * sps)
   }
 }
 
+static void
+h264_sei_message_clear (GstH264SEIMessage * sei_msg)
+{
+  switch (sei_msg->payloadType) {
+    case GST_H264_SEI_REGISTERED_USER_DATA:{
+      GstH264RegisteredUserData *rud = &sei_msg->payload.registered_user_data;
+
+      g_free ((guint8 *) rud->data);
+      rud->data = NULL;
+      break;
+    }
+    default:
+      break;
+  }
+}
+
 /**
  * gst_h264_parser_parse_sei:
  * @nalparser: a #GstH264NalParser
@@ -2299,6 +2364,7 @@ gst_h264_parser_parse_sei (GstH264NalParser * nalparser, GstH264NalUnit * nalu,
   nal_reader_init (&nr, nalu->data + nalu->offset + nalu->header_bytes,
       nalu->size - nalu->header_bytes);
   *messages = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage));
+  g_array_set_clear_func (*messages, (GDestroyNotify) h264_sei_message_clear);
 
   do {
     res = gst_h264_parser_parse_sei_message (nalparser, &nr, &sei);
diff --git a/gst-libs/gst/codecparsers/gsth264parser.h b/gst-libs/gst/codecparsers/gsth264parser.h
index 671b5e5fd..4e063d596 100644
--- a/gst-libs/gst/codecparsers/gsth264parser.h
+++ b/gst-libs/gst/codecparsers/gsth264parser.h
@@ -240,6 +240,7 @@ typedef enum
  * GstH264SEIPayloadType:
  * @GST_H264_SEI_BUF_PERIOD: Buffering Period SEI Message
  * @GST_H264_SEI_PIC_TIMING: Picture Timing SEI Message
+ * @GST_H264_SEI_REGISTERED_USER_DATA: Registered user data (D.2.5)
  * @GST_H264_SEI_RECOVERY_POINT: Recovery Point SEI Message (D.2.7)
  * @GST_H264_SEI_STEREO_VIDEO_INFO: stereo video info SEI message (Since: 1.6)
  * @GST_H264_SEI_FRAME_PACKING: Frame Packing Arrangement (FPA) message that
@@ -252,6 +253,7 @@ typedef enum
 {
   GST_H264_SEI_BUF_PERIOD = 0,
   GST_H264_SEI_PIC_TIMING = 1,
+  GST_H264_SEI_REGISTERED_USER_DATA = 4,
   GST_H264_SEI_RECOVERY_POINT = 6,
   GST_H264_SEI_STEREO_VIDEO_INFO = 21,
   GST_H264_SEI_FRAME_PACKING = 45
@@ -345,6 +347,7 @@ typedef struct _GstH264SliceHdr               GstH264SliceHdr;
 
 typedef struct _GstH264ClockTimestamp         GstH264ClockTimestamp;
 typedef struct _GstH264PicTiming              GstH264PicTiming;
+typedef struct _GstH264RegisteredUserData     GstH264RegisteredUserData;
 typedef struct _GstH264BufferingPeriod        GstH264BufferingPeriod;
 typedef struct _GstH264RecoveryPoint          GstH264RecoveryPoint;
 typedef struct _GstH264StereoVideoInfo        GstH264StereoVideoInfo;
@@ -994,6 +997,14 @@ struct _GstH264PicTiming
   GstH264ClockTimestamp clock_timestamp[3];
 };
 
+struct _GstH264RegisteredUserData
+{
+  guint8 country_code;
+  guint8 country_code_extension;
+  const guint8 *data;
+  guint size;
+};
+
 struct _GstH264BufferingPeriod
 {
   GstH264SPS *sps;
@@ -1022,6 +1033,7 @@ struct _GstH264SEIMessage
   union {
     GstH264BufferingPeriod buffering_period;
     GstH264PicTiming pic_timing;
+    GstH264RegisteredUserData registered_user_data;
     GstH264RecoveryPoint recovery_point;
     GstH264StereoVideoInfo stereo_video_info;
     GstH264FramePacking frame_packing;
diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index c2729102f..c2b3d81e4 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -520,6 +520,95 @@ _nal_name (GstH264NalUnitType nal_type)
 #endif
 
 static void
+gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
+    GstH264RegisteredUserData * rud)
+{
+  guint16 provider_code;
+  guint32 atsc_user_id;
+  GstByteReader br;
+
+  if (rud->country_code != 0xB5)
+    return;
+
+  if (rud->data == NULL || rud->size < 2)
+    return;
+
+  gst_byte_reader_init (&br, rud->data, rud->size);
+
+  provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
+
+  /* There is also 0x29 / 47 for DirecTV, but we don't handle that for now.
+   * https://en.wikipedia.org/wiki/CEA-708#Picture_User_Data */
+  if (provider_code != 0x0031)
+    return;
+
+  /* ANSI/SCTE 128-2010a section 8.1.2 */
+  if (!gst_byte_reader_get_uint32_be (&br, &atsc_user_id))
+    return;
+
+  atsc_user_id = GUINT32_FROM_BE (atsc_user_id);
+  switch (atsc_user_id) {
+    case GST_MAKE_FOURCC ('G', 'A', '9', '4'):{
+      guint8 user_data_type_code, m;
+
+      /* 8.2.1 ATSC1_data() Semantics, Table 15 */
+      if (!gst_byte_reader_get_uint8 (&br, &user_data_type_code))
+        return;
+
+      switch (user_data_type_code) {
+        case 0x03:{
+          guint8 process_cc_data_flag;
+          guint8 cc_count, em_data, b;
+          guint i;
+
+          if (!gst_byte_reader_get_uint8 (&br, &b) || (b & 0x20) != 0)
+            break;
+
+          if (!gst_byte_reader_get_uint8 (&br, &em_data) || em_data != 0xff)
+            break;
+
+          process_cc_data_flag = (b & 0x40) != 0;
+          cc_count = b & 0x1f;
+
+          if (cc_count * 3 > gst_byte_reader_get_remaining (&br))
+            break;
+
+          if (!process_cc_data_flag || cc_count == 0) {
+            gst_byte_reader_skip_unchecked (&br, cc_count * 3);
+            break;
+          }
+
+          /* Shouldn't really happen so let's not go out of our way to handle it */
+          if (h264parse->closedcaptions_size > 0) {
+            GST_WARNING_OBJECT (h264parse, "unused pending closed captions!");
+            GST_MEMDUMP_OBJECT (h264parse, "unused captions being dropped",
+                h264parse->closedcaptions, h264parse->closedcaptions_size);
+          }
+
+          for (i = 0; i < cc_count * 3; i++) {
+            h264parse->closedcaptions[i] =
+                gst_byte_reader_get_uint8_unchecked (&br);
+          }
+          h264parse->closedcaptions_size = cc_count * 3;
+          h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_CEA708_RAW;
+
+          GST_LOG_OBJECT (h264parse, "Extracted closed captions");
+          break;
+        }
+        default:
+          GST_LOG ("Unhandled atsc1 user data type %d", atsc_user_id);
+          break;
+      }
+      if (!gst_byte_reader_get_uint8 (&br, &m) || m != 0xff)
+        GST_WARNING_OBJECT (h264parse, "Marker bits mismatch after ATSC1_data");
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+static void
 gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
 {
   GstH264SEIMessage sei;
@@ -563,6 +652,10 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
         GST_LOG_OBJECT (h264parse, "pic timing updated");
         break;
       }
+      case GST_H264_SEI_REGISTERED_USER_DATA:
+        gst_h264_parse_process_sei_user_data (h264parse,
+            &sei.payload.registered_user_data);
+        break;
       case GST_H264_SEI_BUF_PERIOD:
         if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
             h264parse->dts == GST_CLOCK_TIME_NONE)
@@ -2486,6 +2579,15 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
     buffer = frame->buffer;
   }
 
+  if (h264parse->closedcaptions_size > 0) {
+    gst_buffer_add_video_caption_meta (buffer,
+        h264parse->closedcaptions_type, h264parse->closedcaptions,
+        h264parse->closedcaptions_size);
+
+    h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+    h264parse->closedcaptions_size = 0;
+  }
+
   if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
               &parse->segment, GST_BUFFER_TIMESTAMP (buffer),
               GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {
diff --git a/gst/videoparsers/gsth264parse.h b/gst/videoparsers/gsth264parse.h
index 3d4d8e28d..a8e0120f2 100644
--- a/gst/videoparsers/gsth264parse.h
+++ b/gst/videoparsers/gsth264parse.h
@@ -141,6 +141,11 @@ struct _GstH264Parse
   /* For insertion of AU Delimiter */
   gboolean aud_needed;
   gboolean aud_insert;
+
+  /* pending closed captions */
+  guint8 closedcaptions[96];
+  guint closedcaptions_size;
+  GstVideoCaptionType closedcaptions_type;
 };
 
 struct _GstH264ParseClass