video-sink: add overlay support

5 files changed, 288 insertions, 17 deletions

diff --git a/clutter-gst/clutter-gst-private.h b/clutter-gst/clutter-gst-private.h
index 25dcf5c..c1bbeac 100644
--- a/clutter-gst/clutter-gst-private.h
+++ b/clutter-gst/clutter-gst-private.h
@@ -70,6 +70,10 @@ ClutterGstFrame *clutter_gst_frame_new (void);
 
 ClutterGstFrame *clutter_gst_create_blank_frame (const ClutterColor *color);
 
+ClutterGstOverlay *clutter_gst_overlay_new (void);
+
+ClutterGstOverlays *clutter_gst_overlays_new (void);
+
 void clutter_gst_player_update_frame (ClutterGstPlayer *player,
                                       ClutterGstFrame **frame,
                                       ClutterGstFrame  *new_frame);
@@ -80,6 +84,10 @@ void clutter_gst_frame_update_pixel_aspect_ratio (ClutterGstFrame     *frame,
 void clutter_gst_video_resolution_from_video_info (ClutterGstVideoResolution *resolution,
                                                    GstVideoInfo              *info);
 
+
+gboolean clutter_gst_content_get_paint_frame (ClutterGstContent *content);
+gboolean clutter_gst_content_get_paint_overlays (ClutterGstContent *content);
+
 G_END_DECLS
 
 #endif /* __CLUTTER_GST_PRIVATE_H__ */
diff --git a/clutter-gst/clutter-gst-types.c b/clutter-gst/clutter-gst-types.c
index bdfd827..a4ec6f4 100644
--- a/clutter-gst/clutter-gst-types.c
+++ b/clutter-gst/clutter-gst-types.c
@@ -70,6 +70,97 @@ G_DEFINE_BOXED_TYPE (ClutterGstFrame,
                      clutter_gst_frame_copy,
                      clutter_gst_frame_free);
 
+
+ClutterGstOverlay *
+clutter_gst_overlay_new (void)
+{
+  return g_slice_new0 (ClutterGstOverlay);
+}
+
+static gpointer
+clutter_gst_overlay_copy (gpointer data)
+{
+  if (G_LIKELY (data))
+    {
+      ClutterGstOverlay *overlay = g_slice_dup (ClutterGstOverlay, data);
+
+      if (overlay->pipeline != COGL_INVALID_HANDLE)
+        overlay->pipeline = cogl_object_ref (overlay->pipeline);
+
+      return overlay;
+    }
+
+  return NULL;
+}
+
+static void
+clutter_gst_overlay_free (gpointer data)
+{
+  if (G_LIKELY (data))
+    {
+      ClutterGstOverlay *overlay = (ClutterGstOverlay *) data;
+
+      if (overlay->pipeline != COGL_INVALID_HANDLE)
+        {
+          cogl_object_unref (overlay->pipeline);
+          overlay->pipeline = COGL_INVALID_HANDLE;
+        }
+      g_slice_free (ClutterGstOverlay, overlay);
+    }
+}
+
+G_DEFINE_BOXED_TYPE (ClutterGstOverlay,
+                     clutter_gst_overlay,
+                     clutter_gst_overlay_copy,
+                     clutter_gst_overlay_free);
+
+ClutterGstOverlays *
+clutter_gst_overlays_new (void)
+{
+  ClutterGstOverlays *overlays = g_slice_new0 (ClutterGstOverlays);
+
+  overlays->overlays = g_ptr_array_new_with_free_func (clutter_gst_overlay_free);
+
+  return overlays;
+}
+
+static gpointer
+clutter_gst_overlays_copy (gpointer data)
+{
+  if (G_LIKELY (data))
+    {
+      ClutterGstOverlays *overlays = clutter_gst_overlays_new ();
+      GPtrArray *src_overlays = ((ClutterGstOverlays *) data)->overlays;
+      guint i;
+
+      for (i = 0; i < src_overlays->len; i++)
+        g_ptr_array_add (overlays->overlays,
+                         clutter_gst_overlay_copy (g_ptr_array_index (src_overlays,
+                                                                      i)));
+      return overlays;
+    }
+
+  return NULL;
+}
+
+static void
+clutter_gst_overlays_free (gpointer data)
+{
+  if (G_LIKELY (data))
+    {
+      ClutterGstOverlays *overlays = (ClutterGstOverlays *) data;
+
+      g_ptr_array_unref (overlays->overlays);
+
+      g_slice_free (ClutterGstOverlays, overlays);
+    }
+}
+
+G_DEFINE_BOXED_TYPE (ClutterGstOverlays,
+                     clutter_gst_overlays,
+                     clutter_gst_overlays_copy,
+                     clutter_gst_overlays_free);
+
 static ClutterGstBox *
 clutter_gst_box_copy (const ClutterGstBox *box)
 {
diff --git a/clutter-gst/clutter-gst-types.h b/clutter-gst/clutter-gst-types.h
index 7124d90..b8c7c3e 100644
--- a/clutter-gst/clutter-gst-types.h
+++ b/clutter-gst/clutter-gst-types.h
@@ -36,9 +36,13 @@
 
 #define CLUTTER_GST_TYPE_FRAME            (clutter_gst_frame_get_type ())
 #define CLUTTER_GST_TYPE_BOX              (clutter_gst_box_get_type ())
+#define CLUTTER_GST_TYPE_OVERLAY          (clutter_gst_overlay_get_type ())
+#define CLUTTER_GST_TYPE_OVERLAYS         (clutter_gst_overlays_get_type ())
 
 typedef struct _ClutterGstBox             ClutterGstBox;
 typedef struct _ClutterGstFrame           ClutterGstFrame;
+typedef struct _ClutterGstOverlay         ClutterGstOverlay;
+typedef struct _ClutterGstOverlays        ClutterGstOverlays;
 typedef struct _ClutterGstVideoResolution ClutterGstVideoResolution;
 
 /**
@@ -72,6 +76,25 @@ typedef enum _ClutterGstBufferingMode
 } ClutterGstBufferingMode;
 
 /**
+ * ClutterGstBox:
+ * @x1: X coordinate of the top left corner
+ * @y1: Y coordinate of the top left corner
+ * @x2: X coordinate of the bottom right corner
+ * @y2: Y coordinate of the bottom right corner
+ *
+ * Bounding box of an area in a video texture or actor's allocation.
+ * Coordinates are usually expressed in the [0, 1] interval.
+ */
+struct _ClutterGstBox
+{
+  gfloat x1;
+  gfloat y1;
+
+  gfloat x2;
+  gfloat y2;
+};
+
+/**
  * ClutterGstVideoResolution:
  * @width: the width, in pixels
  * @height: the height, in pixels
@@ -90,7 +113,7 @@ struct _ClutterGstVideoResolution
 /**
  * ClutterGstFrame:
  * @resolution: a #ClutterGstVideoResolution
- * @pipeline: a #CoglHandle to the pipeline to paint a frame
+ * @pipeline: a #CoglPipeline to paint a frame
  *
  * Represents a frame outputted by the #ClutterGstVideoSink.
  */
@@ -100,29 +123,33 @@ struct _ClutterGstFrame
   CoglPipeline              *pipeline;
 };
 
-
 /**
- * ClutterGstBox:
- * @x1: X coordinate of the top left corner
- * @y1: Y coordinate of the top left corner
- * @x2: X coordinate of the bottom right corner
- * @y2: Y coordinate of the bottom right corner
+ * ClutterGstOverlay:
+ * @position: a #ClutterGstBox representing the position of the
+ *            overlay within a #ClutterGstFrame.
+ * @pipeline: a #CoglPipeline to paint an overlay
  *
- * Bounding box of an area in a video texture or actor's allocation.
- * Coordinates are usually expressed in the [0, 1] interval.
+ * Represents a video overlay outputted by the #ClutterGstVideoSink.
  */
-struct _ClutterGstBox
+struct _ClutterGstOverlay
 {
-  gfloat x1;
-  gfloat y1;
-
-  gfloat x2;
-  gfloat y2;
+  ClutterGstBox  position;
+  CoglPipeline  *pipeline;
 };
 
+/**
+ * ClutterGstOverlays:
+ * @overlays: an array of #ClutterGstOverlay
+ */
+struct _ClutterGstOverlays
+{
+  GPtrArray *overlays;
+};
 
-GType clutter_gst_frame_get_type (void) G_GNUC_CONST;
-GType clutter_gst_box_get_type   (void) G_GNUC_CONST;
+GType clutter_gst_frame_get_type     (void) G_GNUC_CONST;
+GType clutter_gst_box_get_type       (void) G_GNUC_CONST;
+GType clutter_gst_overlay_get_type   (void) G_GNUC_CONST;
+GType clutter_gst_overlays_get_type  (void) G_GNUC_CONST;
 
 gfloat clutter_gst_box_get_width     (const ClutterGstBox *box);
 gfloat clutter_gst_box_get_height    (const ClutterGstBox *box);
diff --git a/clutter-gst/clutter-gst-video-sink.c b/clutter-gst/clutter-gst-video-sink.c
index 1959c4e..539319d 100644
--- a/clutter-gst/clutter-gst-video-sink.c
+++ b/clutter-gst/clutter-gst-video-sink.c
@@ -130,6 +130,8 @@ enum
     PIPELINE_READY,
     NEW_FRAME,
 
+    NEW_OVERLAYS,
+
     LAST_SIGNAL
   };
 
@@ -231,8 +233,111 @@ struct _ClutterGstVideoSinkPrivate
   guint8 *tabley;
   guint8 *tableu;
   guint8 *tablev;
+
+  /**/
+  GstVideoOverlayComposition *last_composition;
+  ClutterGstOverlays *overlays;
 };
 
+/* Overlays */
+
+static void
+clutter_gst_video_sink_upload_overlay (ClutterGstVideoSink *sink, GstBuffer *buffer)
+{
+  ClutterGstVideoSinkPrivate *priv = sink->priv;
+
+  GstVideoOverlayComposition *composition = NULL;
+  GstVideoOverlayCompositionMeta *composition_meta;
+  guint i, nb_rectangle;
+
+  composition_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
+  if (composition_meta)
+    composition = composition_meta->overlay;
+
+  if (composition == NULL)
+    {
+      if (priv->last_composition != NULL)
+        {
+          gst_video_overlay_composition_unref (priv->last_composition);
+          priv->last_composition = NULL;
+
+          if (priv->overlays)
+            g_boxed_free (CLUTTER_GST_TYPE_OVERLAYS, priv->overlays);
+          priv->overlays = clutter_gst_overlays_new ();
+
+          g_signal_emit (sink, video_sink_signals[NEW_OVERLAYS], 0);
+        }
+      return;
+    }
+
+  g_clear_pointer (&priv->last_composition, gst_video_overlay_composition_unref);
+  priv->last_composition = gst_video_overlay_composition_ref (composition);
+  if (priv->overlays)
+    g_boxed_free (CLUTTER_GST_TYPE_OVERLAYS, priv->overlays);
+  priv->overlays = clutter_gst_overlays_new ();
+
+  nb_rectangle = gst_video_overlay_composition_n_rectangles (composition);
+  for (i = 0; i < nb_rectangle; i++)
+    {
+      GstVideoOverlayRectangle *rectangle;
+      GstBuffer *comp_buffer;
+      GstMapInfo info;
+      GstVideoMeta *vmeta;
+      gpointer data;
+      gint comp_x, comp_y, stride;
+      guint comp_width, comp_height;
+      CoglTexture *tex;
+      CoglError *error;
+
+      rectangle = gst_video_overlay_composition_get_rectangle (composition, i);
+      comp_buffer =
+        gst_video_overlay_rectangle_get_pixels_unscaled_argb (rectangle,
+                                                              GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
+
+      gst_video_overlay_rectangle_get_render_rectangle (rectangle,
+                                                        &comp_x, &comp_y, &comp_width, &comp_height);
+
+      vmeta = gst_buffer_get_video_meta (comp_buffer);
+      gst_video_meta_map (vmeta, 0, &info, &data, &stride, GST_MAP_READ);
+
+      tex =
+        cogl_texture_2d_new_from_data (priv->ctx,
+                                       comp_width,
+                                       comp_height,
+                                       COGL_PIXEL_FORMAT_BGRA_8888,
+                                       stride, data,
+                                       &error);
+
+      gst_video_meta_unmap (vmeta, 0, &info);
+
+      if (tex != NULL)
+        {
+          ClutterGstOverlay *overlay = clutter_gst_overlay_new ();
+
+          overlay->position.x1 = comp_x;
+          overlay->position.y1 = comp_y;
+          overlay->position.x2 = comp_x + comp_width;
+          overlay->position.y2 = comp_y + comp_height;
+
+          overlay->pipeline = cogl_pipeline_new (priv->ctx);
+          cogl_pipeline_set_layer_texture (overlay->pipeline, 0, tex);
+
+          cogl_object_unref (tex);
+
+          g_ptr_array_add (priv->overlays->overlays, overlay);
+        }
+      else
+        {
+          GST_WARNING_OBJECT (sink,
+                              "Cannot upload overlay texture : %s",
+                              error->message);
+          cogl_error_free (error);
+        }
+    }
+
+  g_signal_emit (sink, video_sink_signals[NEW_OVERLAYS], 0);
+}
+
 /* Snippet cache */
 
 static SnippetCacheEntry *
@@ -1822,6 +1927,8 @@ clutter_gst_source_dispatch (GSource *source,
 
   if (buffer)
     {
+      clutter_gst_video_sink_upload_overlay (gst_source->sink, buffer);
+
       if (gst_buffer_get_video_gl_texture_upload_meta (buffer) != NULL) {
         if (!priv->renderer->upload_gl (gst_source->sink, buffer)) {
           goto fail_upload;
@@ -1919,6 +2026,7 @@ clutter_gst_video_sink_init (ClutterGstVideoSink *sink)
   priv->ctx = clutter_gst_get_cogl_context ();
   priv->renderers = clutter_gst_build_renderers_list (priv->ctx);
   priv->caps = clutter_gst_build_caps (priv->renderers);
+  priv->overlays = clutter_gst_overlays_new ();
 }
 
 static GstFlowReturn
@@ -2092,6 +2200,8 @@ clutter_gst_video_sink_propose_allocation (GstBaseSink *base_sink, GstQuery *que
                                  GST_VIDEO_META_API_TYPE, NULL);
   gst_query_add_allocation_meta (query,
                                  GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, NULL);
+  gst_query_add_allocation_meta (query,
+                                 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
 
   return TRUE;
 }
@@ -2194,6 +2304,29 @@ clutter_gst_video_sink_class_init (ClutterGstVideoSinkClass *klass)
                   g_cclosure_marshal_VOID__VOID,
                   G_TYPE_NONE,
                   0 /* n_params */);
+
+  /**
+   * ClutterGstVideoSink::new-overlays:
+   * @sink: the #ClutterGstVideoSink
+   *
+   * The sink will emit this signal whenever there are new textures
+   * available for set of overlays on the video. After this signal is
+   * emitted, an application can call
+   * clutter_gst_video_sink_get_overlays() to get a set of pipelines
+   * suitable for rendering overlays on a video frame.
+   *
+   * Since: 3.0
+   */
+  video_sink_signals[NEW_OVERLAYS] =
+    g_signal_new ("new-overlays",
+                  CLUTTER_GST_TYPE_VIDEO_SINK,
+                  G_SIGNAL_RUN_LAST,
+                  G_STRUCT_OFFSET (ClutterGstVideoSinkClass, new_overlays),
+                  NULL, /* accumulator */
+                  NULL, /* accu_data */
+                  g_cclosure_marshal_VOID__VOID,
+                  G_TYPE_NONE,
+                  0 /* n_params */);
 }
 
 /**
@@ -2395,3 +2528,12 @@ clutter_gst_video_sink_setup_pipeline (ClutterGstVideoSink *sink,
       priv->renderer->setup_pipeline (sink, priv->pipeline);
     }
 }
+
+
+ClutterGstOverlays *
+clutter_gst_video_sink_get_overlays (ClutterGstVideoSink *sink)
+{
+  g_return_val_if_fail (CLUTTER_GST_IS_VIDEO_SINK (sink), NULL);
+
+  return sink->priv->overlays;
+}
diff --git a/clutter-gst/clutter-gst-video-sink.h b/clutter-gst/clutter-gst-video-sink.h
index 505a6cd..692d3ad 100644
--- a/clutter-gst/clutter-gst-video-sink.h
+++ b/clutter-gst/clutter-gst-video-sink.h
@@ -97,6 +97,7 @@ struct _ClutterGstVideoSinkClass
   /*< public >*/
   void (* new_frame)      (ClutterGstVideoSink *sink);
   void (* pipeline_ready) (ClutterGstVideoSink *sink);
+  void (* new_overlays)   (ClutterGstVideoSink *sink);
 
   /*< private >*/
   void *_padding_dummy[8];
@@ -116,6 +117,8 @@ CoglPipeline *        clutter_gst_video_sink_get_pipeline   (ClutterGstVideoSink
 void                  clutter_gst_video_sink_setup_pipeline (ClutterGstVideoSink *sink,
                                                              CoglPipeline        *pipeline);
 
+ClutterGstOverlays *  clutter_gst_video_sink_get_overlays   (ClutterGstVideoSink *sink);
+
 G_END_DECLS
 
 #endif