From 045a9358712118982fcfe382e2860b29be4d7086 Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Fri, 29 Jan 2016 15:07:59 +1100 Subject: applemedia: refactor GL context code Rework the GL context code. Now both avfvideosrc and vtdec can create an internal GL context for pushing textures. Both elements will still try to use/switch to a local context where available (including after RECONFIGURE events). --- sys/applemedia/Makefile.am | 8 ++- sys/applemedia/avfvideosrc.m | 144 +++++++++++++++------------------------ sys/applemedia/glcontexthelper.c | 132 +++++++++++++++++++++++++++++++++++ sys/applemedia/glcontexthelper.h | 39 +++++++++++ sys/applemedia/vtdec.c | 65 ++++++------------ sys/applemedia/vtdec.h | 2 + 6 files changed, 252 insertions(+), 138 deletions(-) create mode 100644 sys/applemedia/glcontexthelper.c create mode 100644 sys/applemedia/glcontexthelper.h (limited to 'sys') diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am index 2502a9ea3..87b80b5ad 100644 --- a/sys/applemedia/Makefile.am +++ b/sys/applemedia/Makefile.am @@ -6,8 +6,9 @@ libgstapplemedia_la_SOURCES = \ corevideomemory.c \ corevideobuffer.c \ coremediabuffer.c \ - videotexturecache.m \ - atdec.c + videotexturecache.m \ + atdec.c \ + glcontexthelper.c libgstapplemedia_la_CPPFLAGS = \ -Dgst_core_media_buffer_new=gst_core_media_buffer_priv_new \ @@ -74,7 +75,8 @@ noinst_HEADERS = \ videotexturecache.h \ atdec.h \ iosassetsrc.h \ - avfassetsrc.h + avfassetsrc.h \ + glcontexthelper.h if HAVE_IOS diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m index af1122e0a..fe0cb5072 100644 --- a/sys/applemedia/avfvideosrc.m +++ b/sys/applemedia/avfvideosrc.m @@ -22,6 +22,7 @@ #endif #include "avfvideosrc.h" +#include "glcontexthelper.h" #import #if !HAVE_IOS @@ -114,6 +115,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC); BOOL captureScreenMouseClicks; BOOL useVideoMeta; + GstGLContextHelper *ctxh; GstVideoTextureCache *textureCache; } @@ -150,6 +152,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC); - (GstStateChangeReturn)changeState:(GstStateChange)transition; - (GstFlowReturn)create:(GstBuffer **)buf; - (GstCaps *)fixate:(GstCaps *)caps; +- (BOOL)decideAllocation:(GstQuery *)query; - (void)updateStatistics; - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer @@ -180,7 +183,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer captureScreenMouseClicks = NO; useVideoMeta = NO; textureCache = NULL; - + ctxh = gst_gl_context_helper_new (element); mainQueue = dispatch_queue_create ("org.freedesktop.gstreamer.avfvideosrc.main", NULL); workerQueue = @@ -751,24 +754,8 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer dictionaryWithObject:[NSNumber numberWithInt:newformat] forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey]; - if (caps) - gst_caps_unref (caps); - caps = gst_caps_copy (new_caps); - - if (textureCache) - gst_video_texture_cache_free (textureCache); - textureCache = NULL; - - GstCapsFeatures *features = gst_caps_get_features (caps, 0); - if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { - GstGLContext *context = find_gl_context (element); - textureCache = gst_video_texture_cache_new (context); - gst_video_texture_cache_set_format (textureCache, format, caps); - gst_object_unref (context); - } - - GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT - ", pushing textures %d", caps, textureCache != NULL); + gst_caps_replace (&caps, new_caps); + GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT, caps); if (![session isRunning]) [session startRunning]; @@ -808,9 +795,13 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer bufQueue = nil; if (textureCache) - gst_video_texture_cache_free (textureCache); + gst_video_texture_cache_free (textureCache); textureCache = NULL; + if (ctxh) + gst_gl_context_helper_free (ctxh); + ctxh = NULL; + return YES; } @@ -971,81 +962,11 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer return GST_FLOW_OK; } -static GstGLContext * -query_gl_local_context (GstPad *srcpad) -{ - GstGLContext *gl_context = NULL; - GstContext *context = NULL; - GstQuery *query; - - query = gst_query_new_context ("gst.gl.local_context"); - if (gst_pad_peer_query (srcpad, query)) { - gst_query_parse_context (query, &context); - if (context) { - const GstStructure *s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL); - } - } - gst_query_unref (query); - - return gl_context; -} - -static GstGLContext * -find_gl_app_context (GstElement *element) -{ - GstGLContext *gl_context = NULL; - GstContext *context = gst_element_get_context (element, "gst.gl.app_context"); - if (context) { - const GstStructure *s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL); - gst_context_unref (context); - } - - return gl_context; -} - -static GstGLContext * -find_gl_context (GstElement *element) -{ - GstGLContext *gl_context = NULL; - - gl_context = query_gl_local_context (GST_BASE_SRC_PAD (element)); - if (!gl_context) - gl_context = find_gl_app_context (element); - - return gl_context; -} - -static gboolean -caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure, - gpointer user_data) -{ - return !gst_caps_features_contains (features, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -} - - - (GstCaps *)fixate:(GstCaps *)new_caps { - GstGLContext *context; GstStructure *structure; new_caps = gst_caps_make_writable (new_caps); - - context = find_gl_context (element); - if (!context) - gst_caps_filter_and_map_in_place (new_caps, caps_filter_out_gl_memory, NULL); - else - gst_object_unref (context); - - /* this can happen if video/x-raw(memory:GLMemory) is forced but a context is - * not available */ - if (gst_caps_is_empty (new_caps)) { - GST_WARNING_OBJECT (element, "GLMemory requested but no context available"); - return new_caps; - } - new_caps = gst_caps_truncate (new_caps); structure = gst_caps_get_structure (new_caps, 0); /* crank up to 11. This is what the presets do, but we don't use the presets @@ -1056,6 +977,33 @@ caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure, return gst_caps_fixate (new_caps); } +- (BOOL)decideAllocation:(GstQuery *)query +{ + GstCaps *alloc_caps; + GstCapsFeatures *features; + gboolean ret; + + ret = GST_BASE_SRC_CLASS (parent_class)->decide_allocation (baseSrc, query); + if (!ret) + return ret; + + gst_query_parse_allocation (query, &alloc_caps, NULL); + features = gst_caps_get_features (alloc_caps, 0); + if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { + gst_gl_context_helper_ensure_context (ctxh); + GST_INFO_OBJECT (element, "pushing textures, context %p old context %p", + ctxh->context, textureCache ? textureCache->ctx : NULL); + if (textureCache && textureCache->ctx != ctxh->context) { + gst_video_texture_cache_free (textureCache); + textureCache = NULL; + } + textureCache = gst_video_texture_cache_new (ctxh->context); + gst_video_texture_cache_set_format (textureCache, format, alloc_caps); + } + + return TRUE; +} + - (void)getSampleBuffer:(CMSampleBufferRef)sbuf timestamp:(GstClockTime *)outTimestamp duration:(GstClockTime *)outDuration @@ -1186,6 +1134,8 @@ static GstFlowReturn gst_avf_video_src_create (GstPushSrc * pushsrc, GstBuffer ** buf); static GstCaps * gst_avf_video_src_fixate (GstBaseSrc * bsrc, GstCaps * caps); +static gboolean gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc, + GstQuery * query); static void gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass) @@ -1209,6 +1159,7 @@ gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass) gstbasesrc_class->unlock = gst_avf_video_src_unlock; gstbasesrc_class->unlock_stop = gst_avf_video_src_unlock_stop; gstbasesrc_class->fixate = gst_avf_video_src_fixate; + gstbasesrc_class->decide_allocation = gst_avf_video_src_decide_allocation; gstpushsrc_class->create = gst_avf_video_src_create; @@ -1463,3 +1414,16 @@ gst_avf_video_src_fixate (GstBaseSrc * bsrc, GstCaps * caps) return ret; } + +static gboolean +gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc, + GstQuery * query) +{ + gboolean ret; + + OBJC_CALLOUT_BEGIN (); + ret = [GST_AVF_VIDEO_SRC_IMPL (bsrc) decideAllocation:query]; + OBJC_CALLOUT_END (); + + return ret; +} diff --git a/sys/applemedia/glcontexthelper.c b/sys/applemedia/glcontexthelper.c new file mode 100644 index 000000000..4e6f6175e --- /dev/null +++ b/sys/applemedia/glcontexthelper.c @@ -0,0 +1,132 @@ +/* GStreamer + * Copyright (C) 2016 Alessandro Decina + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "glcontexthelper.h" + +static GstGLContext * +_find_local_gl_context (GstGLContextHelper * ctxh) +{ + GstQuery *query; + GstContext *context; + GstGLContext *gl_context = NULL; + const GstStructure *s; + + g_return_val_if_fail (ctxh != NULL, FALSE); + + query = gst_query_new_context ("gst.gl.local_context"); + if (gst_gl_run_query (ctxh->element, query, GST_PAD_SRC)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL); + } + } + if (!gl_context && gst_gl_run_query (ctxh->element, query, GST_PAD_SINK)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL); + } + } + + GST_DEBUG_OBJECT (ctxh->element, "found local context %p", gl_context); + + gst_query_unref (query); + + return gl_context; +} + +GstGLContextHelper * +gst_gl_context_helper_new (GstElement * element) +{ + GstGLContextHelper *ctxh = g_new0 (GstGLContextHelper, 1); + ctxh->element = gst_object_ref (element); + + return ctxh; +} + +void +gst_gl_context_helper_free (GstGLContextHelper * ctxh) +{ + g_return_if_fail (ctxh != NULL); + + gst_object_unref (ctxh->element); + + if (ctxh->display) + gst_object_unref (ctxh->display); + + if (ctxh->context) + gst_object_unref (ctxh->context); + + if (ctxh->other_context) + gst_object_unref (ctxh->other_context); + + g_free (ctxh); +} + +void +gst_gl_context_helper_ensure_context (GstGLContextHelper * ctxh) +{ + GError *error = NULL; + GstGLContext *context; + + g_return_if_fail (ctxh != NULL); + + if (!ctxh->display) + gst_gl_ensure_element_data (ctxh->element, &ctxh->display, + &ctxh->other_context); + + context = _find_local_gl_context (ctxh); + if (context) { + GST_INFO_OBJECT (ctxh->element, "found local context %p, old context %p", + context, ctxh->context); + if (ctxh->context) + gst_object_unref (ctxh->context); + ctxh->context = context; + } + + if (!ctxh->context) { + GST_OBJECT_LOCK (ctxh->display); + do { + if (ctxh->context) + gst_object_unref (ctxh->context); + ctxh->context = + gst_gl_display_get_gl_context_for_thread (ctxh->display, NULL); + if (!ctxh->context) { + if (!gst_gl_display_create_context (ctxh->display, + ctxh->other_context, &ctxh->context, &error)) { + GST_OBJECT_UNLOCK (ctxh->display); + goto context_error; + } + } + } while (!gst_gl_display_add_context (ctxh->display, ctxh->context)); + GST_OBJECT_UNLOCK (ctxh->display); + } + + return; + +context_error: + { + GST_ELEMENT_ERROR (ctxh->element, RESOURCE, NOT_FOUND, ("%s", + error->message), (NULL)); + g_clear_error (&error); + + return; + } +} diff --git a/sys/applemedia/glcontexthelper.h b/sys/applemedia/glcontexthelper.h new file mode 100644 index 000000000..4e043f103 --- /dev/null +++ b/sys/applemedia/glcontexthelper.h @@ -0,0 +1,39 @@ +/* GStreamer + * Copyright (C) 2016 Alessandro Decina + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _GST_GL_CONTEXT_HELPER_H_ +#define _GST_GL_CONTEXT_HELPER_H_ + +#include +#include + +typedef struct _GstGLContextHelper +{ + GstElement *element; + GstGLDisplay *display; + GstGLContext *context; + GstGLContext *other_context; +} GstGLContextHelper; + +GstGLContextHelper * gst_gl_context_helper_new (GstElement *element); +void gst_gl_context_helper_free (GstGLContextHelper *ctxh); +void gst_gl_context_helper_ensure_context (GstGLContextHelper *ctxh); + +#endif + diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c index ca8953e31..81ccb6602 100644 --- a/sys/applemedia/vtdec.c +++ b/sys/applemedia/vtdec.c @@ -152,6 +152,7 @@ static void gst_vtdec_init (GstVtdec * vtdec) { vtdec->reorder_queue = g_async_queue_new (); + vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec)); } void @@ -162,7 +163,7 @@ gst_vtdec_finalize (GObject * object) GST_DEBUG_OBJECT (vtdec, "finalize"); g_async_queue_unref (vtdec->reorder_queue); - + gst_gl_context_helper_free (vtdec->ctxh); G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object); } @@ -198,26 +199,6 @@ gst_vtdec_stop (GstVideoDecoder * decoder) return TRUE; } -static GstGLContext * -query_gl_context (GstVtdec * vtdec) -{ - GstGLContext *gl_context = NULL; - GstContext *context = NULL; - GstQuery *query; - - query = gst_query_new_context ("gst.gl.local_context"); - if (gst_pad_peer_query (GST_VIDEO_DECODER_SRC_PAD (vtdec), query)) { - gst_query_parse_context (query, &context); - if (context) { - const GstStructure *s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL); - } - } - gst_query_unref (query); - - return gl_context; -} - static void setup_texture_cache (GstVtdec * vtdec, GstGLContext * context) { @@ -225,8 +206,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context) g_return_if_fail (vtdec->texture_cache == NULL); - GST_INFO_OBJECT (vtdec, "Setting up texture cache. GL context %p", context); - output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec)); vtdec->texture_cache = gst_video_texture_cache_new (context); gst_video_texture_cache_set_format (vtdec->texture_cache, @@ -234,14 +213,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context) gst_video_codec_state_unref (output_state); } -static gboolean -caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure, - gpointer user_data) -{ - return !gst_caps_features_contains (features, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -} - static gboolean gst_vtdec_negotiate (GstVideoDecoder * decoder) { @@ -250,7 +221,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) GstVideoFormat format; GstStructure *structure; const gchar *s; - GstGLContext *context; GstVtdec *vtdec; gboolean ret = TRUE; GstCapsFeatures *features = NULL; @@ -263,9 +233,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) gst_caps_make_writable (gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), templcaps)); gst_caps_unref (templcaps); - context = query_gl_context (vtdec); - if (!context) - gst_caps_filter_and_map_in_place (caps, caps_filter_out_gl_memory, NULL); caps = gst_caps_truncate (caps); structure = gst_caps_get_structure (caps, 0); @@ -306,21 +273,29 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) } ret = gst_vtdec_create_session (vtdec, format); - if (ret) { - if (vtdec->texture_cache) { - gst_video_texture_cache_free (vtdec->texture_cache); - vtdec->texture_cache = NULL; - } - - if (output_textures) - setup_texture_cache (vtdec, context); + } + + if (ret && output_textures) { + /* call this regardless of whether caps have changed or not since a new + * local context could have become available + */ + gst_gl_context_helper_ensure_context (vtdec->ctxh); + + GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p", + vtdec->ctxh->context, + vtdec->texture_cache ? vtdec->texture_cache->ctx : NULL); + + if (vtdec->texture_cache + && vtdec->texture_cache->ctx != vtdec->ctxh->context) { + gst_video_texture_cache_free (vtdec->texture_cache); + vtdec->texture_cache = NULL; } + if (!vtdec->texture_cache) + setup_texture_cache (vtdec, vtdec->ctxh->context); } if (prevcaps) gst_caps_unref (prevcaps); - if (context) - gst_object_unref (context); if (!ret) return ret; diff --git a/sys/applemedia/vtdec.h b/sys/applemedia/vtdec.h index 7d194f4a7..46354d8a8 100644 --- a/sys/applemedia/vtdec.h +++ b/sys/applemedia/vtdec.h @@ -26,6 +26,7 @@ #include #include #include "videotexturecache.h" +#include "glcontexthelper.h" G_BEGIN_DECLS @@ -48,6 +49,7 @@ struct _GstVtdec GAsyncQueue *reorder_queue; gint reorder_queue_length; GstVideoTextureCache *texture_cache; + GstGLContextHelper *ctxh; gboolean require_hardware; }; -- cgit v1.2.1