applemedia: refactor GL context code

Rework the GL context code. Now both avfvideosrc and vtdec can create an
internal GL context for pushing textures. Both elements will still try to
use/switch to a local context where available (including after RECONFIGURE
events).

6 files changed, 252 insertions, 138 deletions

diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am
index 2502a9ea3..87b80b5ad 100644
--- a/sys/applemedia/Makefile.am
+++ b/sys/applemedia/Makefile.am
@@ -6,8 +6,9 @@ libgstapplemedia_la_SOURCES =			\
 	corevideomemory.c			\
 	corevideobuffer.c			\
 	coremediabuffer.c			\
-	videotexturecache.m 		\
-	atdec.c
+	videotexturecache.m 			\
+	atdec.c 				\
+	glcontexthelper.c
 
 libgstapplemedia_la_CPPFLAGS =			\
 	-Dgst_core_media_buffer_new=gst_core_media_buffer_priv_new		\
@@ -74,7 +75,8 @@ noinst_HEADERS =				\
 	videotexturecache.h			\
 	atdec.h 				\
 	iosassetsrc.h				\
-	avfassetsrc.h
+	avfassetsrc.h  				\
+	glcontexthelper.h
 
 if HAVE_IOS
 
diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m
index af1122e0a..fe0cb5072 100644
--- a/sys/applemedia/avfvideosrc.m
+++ b/sys/applemedia/avfvideosrc.m
@@ -22,6 +22,7 @@
 #endif
 
 #include "avfvideosrc.h"
+#include "glcontexthelper.h"
 
 #import <AVFoundation/AVFoundation.h>
 #if !HAVE_IOS
@@ -114,6 +115,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
   BOOL captureScreenMouseClicks;
 
   BOOL useVideoMeta;
+  GstGLContextHelper *ctxh;
   GstVideoTextureCache *textureCache;
 }
 
@@ -150,6 +152,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
 - (GstStateChangeReturn)changeState:(GstStateChange)transition;
 - (GstFlowReturn)create:(GstBuffer **)buf;
 - (GstCaps *)fixate:(GstCaps *)caps;
+- (BOOL)decideAllocation:(GstQuery *)query;
 - (void)updateStatistics;
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
 didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
@@ -180,7 +183,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
     captureScreenMouseClicks = NO;
     useVideoMeta = NO;
     textureCache = NULL;
-
+    ctxh = gst_gl_context_helper_new (element);
     mainQueue =
         dispatch_queue_create ("org.freedesktop.gstreamer.avfvideosrc.main", NULL);
     workerQueue =
@@ -751,24 +754,8 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
         dictionaryWithObject:[NSNumber numberWithInt:newformat]
         forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
 
-    if (caps)
-      gst_caps_unref (caps);
-    caps = gst_caps_copy (new_caps);
-
-    if (textureCache)
-      gst_video_texture_cache_free (textureCache);
-    textureCache = NULL;
-
-    GstCapsFeatures *features = gst_caps_get_features (caps, 0);
-    if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
-      GstGLContext *context = find_gl_context (element);
-      textureCache = gst_video_texture_cache_new (context);
-      gst_video_texture_cache_set_format (textureCache, format, caps);
-      gst_object_unref (context);
-    }
-
-    GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT
-        ", pushing textures %d", caps, textureCache != NULL);
+    gst_caps_replace (&caps, new_caps);
+    GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT, caps);
 
     if (![session isRunning])
       [session startRunning];
@@ -808,9 +795,13 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
   bufQueue = nil;
 
   if (textureCache)
-      gst_video_texture_cache_free (textureCache);
+    gst_video_texture_cache_free (textureCache);
   textureCache = NULL;
 
+  if (ctxh)
+    gst_gl_context_helper_free (ctxh);
+  ctxh = NULL;
+
   return YES;
 }
 
@@ -971,81 +962,11 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
   return GST_FLOW_OK;
 }
 
-static GstGLContext *
-query_gl_local_context (GstPad *srcpad)
-{
-  GstGLContext *gl_context = NULL;
-  GstContext *context = NULL;
-  GstQuery *query;
-
-  query = gst_query_new_context ("gst.gl.local_context");
-  if (gst_pad_peer_query (srcpad, query)) {
-    gst_query_parse_context (query, &context);
-    if (context) {
-      const GstStructure *s = gst_context_get_structure (context);
-      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
-    }
-  }
-  gst_query_unref (query);
-
-  return gl_context;
-}
-
-static GstGLContext *
-find_gl_app_context (GstElement *element)
-{
-  GstGLContext *gl_context = NULL;
-  GstContext *context = gst_element_get_context (element, "gst.gl.app_context");
-  if (context) {
-    const GstStructure *s = gst_context_get_structure (context);
-    gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
-    gst_context_unref (context);
-  }
-
-  return gl_context;
-}
-
-static GstGLContext *
-find_gl_context (GstElement *element)
-{
-  GstGLContext *gl_context = NULL;
-
-  gl_context = query_gl_local_context (GST_BASE_SRC_PAD (element));
-  if (!gl_context)
-    gl_context = find_gl_app_context (element);
-
-  return gl_context;
-}
-
-static gboolean
-caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
-    gpointer user_data)
-{
-  return !gst_caps_features_contains (features,
-      GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
-}
-
-
 - (GstCaps *)fixate:(GstCaps *)new_caps
 {
-  GstGLContext *context;
   GstStructure *structure;
 
   new_caps = gst_caps_make_writable (new_caps);
-
-  context = find_gl_context (element);
-  if (!context)
-    gst_caps_filter_and_map_in_place (new_caps, caps_filter_out_gl_memory, NULL);
-  else
-    gst_object_unref (context);
-
-  /* this can happen if video/x-raw(memory:GLMemory) is forced but a context is
-   * not available */
-  if (gst_caps_is_empty (new_caps)) {
-    GST_WARNING_OBJECT (element, "GLMemory requested but no context available");
-    return new_caps;
-  }
-
   new_caps = gst_caps_truncate (new_caps);
   structure = gst_caps_get_structure (new_caps, 0);
   /* crank up to 11. This is what the presets do, but we don't use the presets
@@ -1056,6 +977,33 @@ caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
   return gst_caps_fixate (new_caps);
 }
 
+- (BOOL)decideAllocation:(GstQuery *)query
+{
+  GstCaps *alloc_caps;
+  GstCapsFeatures *features;
+  gboolean ret;
+
+  ret = GST_BASE_SRC_CLASS (parent_class)->decide_allocation (baseSrc, query);
+  if (!ret)
+    return ret;
+
+  gst_query_parse_allocation (query, &alloc_caps, NULL);
+  features = gst_caps_get_features (alloc_caps, 0);
+  if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
+    gst_gl_context_helper_ensure_context (ctxh);
+    GST_INFO_OBJECT (element, "pushing textures, context %p old context %p",
+        ctxh->context, textureCache ? textureCache->ctx : NULL);
+    if (textureCache && textureCache->ctx != ctxh->context) {
+      gst_video_texture_cache_free (textureCache);
+      textureCache = NULL;
+    }
+    textureCache = gst_video_texture_cache_new (ctxh->context);
+    gst_video_texture_cache_set_format (textureCache, format, alloc_caps);
+  }
+
+  return TRUE;
+}
+
 - (void)getSampleBuffer:(CMSampleBufferRef)sbuf
               timestamp:(GstClockTime *)outTimestamp
                duration:(GstClockTime *)outDuration
@@ -1186,6 +1134,8 @@ static GstFlowReturn gst_avf_video_src_create (GstPushSrc * pushsrc,
     GstBuffer ** buf);
 static GstCaps * gst_avf_video_src_fixate (GstBaseSrc * bsrc,
     GstCaps * caps);
+static gboolean gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc,
+    GstQuery * query);
 
 static void
 gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass)
@@ -1209,6 +1159,7 @@ gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass)
   gstbasesrc_class->unlock = gst_avf_video_src_unlock;
   gstbasesrc_class->unlock_stop = gst_avf_video_src_unlock_stop;
   gstbasesrc_class->fixate = gst_avf_video_src_fixate;
+  gstbasesrc_class->decide_allocation = gst_avf_video_src_decide_allocation;
 
   gstpushsrc_class->create = gst_avf_video_src_create;
 
@@ -1463,3 +1414,16 @@ gst_avf_video_src_fixate (GstBaseSrc * bsrc, GstCaps * caps)
 
   return ret;
 }
+
+static gboolean
+gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc,
+    GstQuery * query)
+{
+  gboolean ret;
+
+  OBJC_CALLOUT_BEGIN ();
+  ret = [GST_AVF_VIDEO_SRC_IMPL (bsrc) decideAllocation:query];
+  OBJC_CALLOUT_END ();
+
+  return ret;
+}
diff --git a/sys/applemedia/glcontexthelper.c b/sys/applemedia/glcontexthelper.c
new file mode 100644
index 000000000..4e6f6175e
--- /dev/null
+++ b/sys/applemedia/glcontexthelper.c
@@ -0,0 +1,132 @@
+/* GStreamer
+ * Copyright (C) 2016 Alessandro Decina <alessandro.d@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include "glcontexthelper.h"
+
+static GstGLContext *
+_find_local_gl_context (GstGLContextHelper * ctxh)
+{
+  GstQuery *query;
+  GstContext *context;
+  GstGLContext *gl_context = NULL;
+  const GstStructure *s;
+
+  g_return_val_if_fail (ctxh != NULL, FALSE);
+
+  query = gst_query_new_context ("gst.gl.local_context");
+  if (gst_gl_run_query (ctxh->element, query, GST_PAD_SRC)) {
+    gst_query_parse_context (query, &context);
+    if (context) {
+      s = gst_context_get_structure (context);
+      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
+    }
+  }
+  if (!gl_context && gst_gl_run_query (ctxh->element, query, GST_PAD_SINK)) {
+    gst_query_parse_context (query, &context);
+    if (context) {
+      s = gst_context_get_structure (context);
+      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
+    }
+  }
+
+  GST_DEBUG_OBJECT (ctxh->element, "found local context %p", gl_context);
+
+  gst_query_unref (query);
+
+  return gl_context;
+}
+
+GstGLContextHelper *
+gst_gl_context_helper_new (GstElement * element)
+{
+  GstGLContextHelper *ctxh = g_new0 (GstGLContextHelper, 1);
+  ctxh->element = gst_object_ref (element);
+
+  return ctxh;
+}
+
+void
+gst_gl_context_helper_free (GstGLContextHelper * ctxh)
+{
+  g_return_if_fail (ctxh != NULL);
+
+  gst_object_unref (ctxh->element);
+
+  if (ctxh->display)
+    gst_object_unref (ctxh->display);
+
+  if (ctxh->context)
+    gst_object_unref (ctxh->context);
+
+  if (ctxh->other_context)
+    gst_object_unref (ctxh->other_context);
+
+  g_free (ctxh);
+}
+
+void
+gst_gl_context_helper_ensure_context (GstGLContextHelper * ctxh)
+{
+  GError *error = NULL;
+  GstGLContext *context;
+
+  g_return_if_fail (ctxh != NULL);
+
+  if (!ctxh->display)
+    gst_gl_ensure_element_data (ctxh->element, &ctxh->display,
+        &ctxh->other_context);
+
+  context = _find_local_gl_context (ctxh);
+  if (context) {
+    GST_INFO_OBJECT (ctxh->element, "found local context %p, old context %p",
+        context, ctxh->context);
+    if (ctxh->context)
+      gst_object_unref (ctxh->context);
+    ctxh->context = context;
+  }
+
+  if (!ctxh->context) {
+    GST_OBJECT_LOCK (ctxh->display);
+    do {
+      if (ctxh->context)
+        gst_object_unref (ctxh->context);
+      ctxh->context =
+          gst_gl_display_get_gl_context_for_thread (ctxh->display, NULL);
+      if (!ctxh->context) {
+        if (!gst_gl_display_create_context (ctxh->display,
+                ctxh->other_context, &ctxh->context, &error)) {
+          GST_OBJECT_UNLOCK (ctxh->display);
+          goto context_error;
+        }
+      }
+    } while (!gst_gl_display_add_context (ctxh->display, ctxh->context));
+    GST_OBJECT_UNLOCK (ctxh->display);
+  }
+
+  return;
+
+context_error:
+  {
+    GST_ELEMENT_ERROR (ctxh->element, RESOURCE, NOT_FOUND, ("%s",
+            error->message), (NULL));
+    g_clear_error (&error);
+
+    return;
+  }
+}
diff --git a/sys/applemedia/glcontexthelper.h b/sys/applemedia/glcontexthelper.h
new file mode 100644
index 000000000..4e043f103
--- /dev/null
+++ b/sys/applemedia/glcontexthelper.h
@@ -0,0 +1,39 @@
+/* GStreamer
+ * Copyright (C) 2016 Alessandro Decina <alessandro.d@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_GL_CONTEXT_HELPER_H_
+#define _GST_GL_CONTEXT_HELPER_H_
+
+#include <gst/gst.h>
+#include <gst/gl/gl.h>
+
+typedef struct _GstGLContextHelper
+{
+  GstElement *element;
+  GstGLDisplay *display;
+  GstGLContext *context;
+  GstGLContext *other_context;
+} GstGLContextHelper;
+
+GstGLContextHelper * gst_gl_context_helper_new (GstElement *element);
+void gst_gl_context_helper_free (GstGLContextHelper *ctxh);
+void gst_gl_context_helper_ensure_context (GstGLContextHelper *ctxh);
+
+#endif
+
diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c
index ca8953e31..81ccb6602 100644
--- a/sys/applemedia/vtdec.c
+++ b/sys/applemedia/vtdec.c
@@ -152,6 +152,7 @@ static void
 gst_vtdec_init (GstVtdec * vtdec)
 {
   vtdec->reorder_queue = g_async_queue_new ();
+  vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
 }
 
 void
@@ -162,7 +163,7 @@ gst_vtdec_finalize (GObject * object)
   GST_DEBUG_OBJECT (vtdec, "finalize");
 
   g_async_queue_unref (vtdec->reorder_queue);
-
+  gst_gl_context_helper_free (vtdec->ctxh);
 
   G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
 }
@@ -198,26 +199,6 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
   return TRUE;
 }
 
-static GstGLContext *
-query_gl_context (GstVtdec * vtdec)
-{
-  GstGLContext *gl_context = NULL;
-  GstContext *context = NULL;
-  GstQuery *query;
-
-  query = gst_query_new_context ("gst.gl.local_context");
-  if (gst_pad_peer_query (GST_VIDEO_DECODER_SRC_PAD (vtdec), query)) {
-    gst_query_parse_context (query, &context);
-    if (context) {
-      const GstStructure *s = gst_context_get_structure (context);
-      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
-    }
-  }
-  gst_query_unref (query);
-
-  return gl_context;
-}
-
 static void
 setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
 {
@@ -225,8 +206,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
 
   g_return_if_fail (vtdec->texture_cache == NULL);
 
-  GST_INFO_OBJECT (vtdec, "Setting up texture cache. GL context %p", context);
-
   output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
   vtdec->texture_cache = gst_video_texture_cache_new (context);
   gst_video_texture_cache_set_format (vtdec->texture_cache,
@@ -235,14 +214,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
 }
 
 static gboolean
-caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
-    gpointer user_data)
-{
-  return !gst_caps_features_contains (features,
-      GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
-}
-
-static gboolean
 gst_vtdec_negotiate (GstVideoDecoder * decoder)
 {
   GstVideoCodecState *output_state = NULL;
@@ -250,7 +221,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
   GstVideoFormat format;
   GstStructure *structure;
   const gchar *s;
-  GstGLContext *context;
   GstVtdec *vtdec;
   gboolean ret = TRUE;
   GstCapsFeatures *features = NULL;
@@ -263,9 +233,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
       gst_caps_make_writable (gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD
           (vtdec), templcaps));
   gst_caps_unref (templcaps);
-  context = query_gl_context (vtdec);
-  if (!context)
-    gst_caps_filter_and_map_in_place (caps, caps_filter_out_gl_memory, NULL);
 
   caps = gst_caps_truncate (caps);
   structure = gst_caps_get_structure (caps, 0);
@@ -306,21 +273,29 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
     }
 
     ret = gst_vtdec_create_session (vtdec, format);
-    if (ret) {
-      if (vtdec->texture_cache) {
-        gst_video_texture_cache_free (vtdec->texture_cache);
-        vtdec->texture_cache = NULL;
-      }
-
-      if (output_textures)
-        setup_texture_cache (vtdec, context);
+  }
+
+  if (ret && output_textures) {
+    /* call this regardless of whether caps have changed or not since a new
+     * local context could have become available
+     */
+    gst_gl_context_helper_ensure_context (vtdec->ctxh);
+
+    GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
+        vtdec->ctxh->context,
+        vtdec->texture_cache ? vtdec->texture_cache->ctx : NULL);
+
+    if (vtdec->texture_cache
+        && vtdec->texture_cache->ctx != vtdec->ctxh->context) {
+      gst_video_texture_cache_free (vtdec->texture_cache);
+      vtdec->texture_cache = NULL;
     }
+    if (!vtdec->texture_cache)
+      setup_texture_cache (vtdec, vtdec->ctxh->context);
   }
 
   if (prevcaps)
     gst_caps_unref (prevcaps);
-  if (context)
-    gst_object_unref (context);
 
   if (!ret)
     return ret;
diff --git a/sys/applemedia/vtdec.h b/sys/applemedia/vtdec.h
index 7d194f4a7..46354d8a8 100644
--- a/sys/applemedia/vtdec.h
+++ b/sys/applemedia/vtdec.h
@@ -26,6 +26,7 @@
 #include <CoreMedia/CoreMedia.h>
 #include <VideoToolbox/VideoToolbox.h>
 #include "videotexturecache.h"
+#include "glcontexthelper.h"
 
 G_BEGIN_DECLS
 
@@ -48,6 +49,7 @@ struct _GstVtdec
   GAsyncQueue *reorder_queue;
   gint reorder_queue_length;
   GstVideoTextureCache *texture_cache;
+  GstGLContextHelper *ctxh;
 
   gboolean require_hardware;
 };