diff options
author | Benjamin Otte <otte@redhat.com> | 2018-03-18 03:22:32 +0100 |
---|---|---|
committer | Benjamin Otte <otte@redhat.com> | 2018-03-18 05:41:29 +0100 |
commit | c45ec8e0d2b6ad9e56c296bea297d3f01ba5556c (patch) | |
tree | e8752b0129fd1919a121123abed14104028cecc4 | |
parent | fa62485cf038e3ff7fadb74a27e5287c80ebcb6e (diff) | |
download | gtk+-wip/otte/paintable.tar.gz |
media: Add support for OpenGL to GtkGstMediaFilewip/otte/paintable
-rw-r--r-- | meson.build | 2 | ||||
-rw-r--r-- | modules/media/gtkgstmediafile.c | 20 | ||||
-rw-r--r-- | modules/media/gtkgstpaintable.c | 53 | ||||
-rw-r--r-- | modules/media/gtkgstpaintableprivate.h | 5 | ||||
-rw-r--r-- | modules/media/gtkgstsink.c | 324 | ||||
-rw-r--r-- | modules/media/gtkgstsinkprivate.h | 7 | ||||
-rw-r--r-- | modules/media/meson.build | 2 |
7 files changed, 379 insertions, 34 deletions
diff --git a/meson.build b/meson.build index a5f6a4ddde..31568fd476 100644 --- a/meson.build +++ b/meson.build @@ -614,6 +614,8 @@ endif gstreamer_enabled = get_option('gstreamer') if gstreamer_enabled gstplayer_dep = dependency('gstreamer-player-1.0', version: '>= 1.12.3', required: true) + gstgl_dep = dependency('gstreamer-gl-1.0', version: '>= 1.12.3', required: true) + gstreamer_deps = [gstplayer_dep, gstgl_dep] cdata.set('HAVE_GSTREAMER', 1) media_backends += ['gstreamer'] endif diff --git a/modules/media/gtkgstmediafile.c b/modules/media/gtkgstmediafile.c index 93162eae5b..50bae088ae 100644 --- a/modules/media/gtkgstmediafile.c +++ b/modules/media/gtkgstmediafile.c @@ -279,6 +279,24 @@ gtk_gst_media_file_update_audio (GtkMediaStream *stream, } static void +gtk_gst_media_stream_realize (GtkMediaStream *stream, + GdkWindow *window) +{ + GtkGstMediaFile *self = GTK_GST_MEDIA_FILE (stream); + + gtk_gst_paintable_realize (GTK_GST_PAINTABLE (self->paintable), window); +} + +static void +gtk_gst_media_stream_unrealize (GtkMediaStream *stream, + GdkWindow *window) +{ + GtkGstMediaFile *self = GTK_GST_MEDIA_FILE (stream); + + gtk_gst_paintable_unrealize (GTK_GST_PAINTABLE (self->paintable), window); +} + +static void gtk_gst_media_file_dispose (GObject *object) { GtkGstMediaFile *self = GTK_GST_MEDIA_FILE (object); @@ -308,6 +326,8 @@ gtk_gst_media_file_class_init (GtkGstMediaFileClass *klass) stream_class->pause = gtk_gst_media_file_pause; stream_class->seek = gtk_gst_media_file_seek; stream_class->update_audio = gtk_gst_media_file_update_audio; + stream_class->realize = gtk_gst_media_stream_realize; + stream_class->unrealize = gtk_gst_media_stream_unrealize; gobject_class->dispose = gtk_gst_media_file_dispose; } diff --git a/modules/media/gtkgstpaintable.c b/modules/media/gtkgstpaintable.c index 980459a34d..020e175b0e 100644 --- a/modules/media/gtkgstpaintable.c +++ b/modules/media/gtkgstpaintable.c @@ -29,6 +29,8 @@ struct _GtkGstPaintable { GObject parent_instance; + GdkGLContext *context; + GdkPaintable *image; }; @@ -109,10 +111,14 @@ gtk_gst_paintable_video_renderer_create_video_sink (GstPlayerVideoRenderer *rend GstPlayer *player) { GtkGstPaintable *self = GTK_GST_PAINTABLE (renderer); + GstElement *element; + + element = g_object_new (GTK_TYPE_GST_SINK, + "paintable", self, + "gl-context", self->context, + NULL); - return g_object_new (GTK_TYPE_GST_SINK, - "paintable", self, - NULL); + return element; } static void @@ -156,6 +162,47 @@ gtk_gst_paintable_new (void) return g_object_new (GTK_TYPE_GST_PAINTABLE, NULL); } +void +gtk_gst_paintable_realize (GtkGstPaintable *self, + GdkWindow *window) +{ + GError *error = NULL; + + if (self->context) + return; + + self->context = gdk_window_create_gl_context (window, &error); + if (self->context == NULL) + { + GST_INFO ("failed to create GDK GL context: %s", error->message); + g_error_free (error); + return; + } + + if (!gdk_gl_context_realize (self->context, &error)) + { + GST_INFO ("failed to realize GDK GL context: %s", error->message); + g_clear_object (&self->context); + g_error_free (error); + return; + } +} + +void +gtk_gst_paintable_unrealize (GtkGstPaintable *self, + GdkWindow *window) +{ + /* XXX: We could be smarter here and: + * - track how often we were realized with that window + * - track alternate windows + */ + if (self->context == NULL) + return; + + if (gdk_gl_context_get_window (self->context) == window) + g_clear_object (&self->context); +} + static void gtk_gst_paintable_set_paintable (GtkGstPaintable *self, GdkPaintable *paintable) diff --git a/modules/media/gtkgstpaintableprivate.h b/modules/media/gtkgstpaintableprivate.h index 7fc2620f9b..155d076db6 100644 --- a/modules/media/gtkgstpaintableprivate.h +++ b/modules/media/gtkgstpaintableprivate.h @@ -30,6 +30,11 @@ G_DECLARE_FINAL_TYPE (GtkGstPaintable, gtk_gst_paintable, GTK, GST_PAINTABLE, GO GdkPaintable * gtk_gst_paintable_new (void); +void gtk_gst_paintable_realize (GtkGstPaintable *self, + GdkWindow *window); +void gtk_gst_paintable_unrealize (GtkGstPaintable *self, + GdkWindow *window); + void gtk_gst_paintable_queue_set_texture (GtkGstPaintable *self, GdkTexture *texture); diff --git a/modules/media/gtkgstsink.c b/modules/media/gtkgstsink.c index da11cbd854..d8ca3b7b26 100644 --- a/modules/media/gtkgstsink.c +++ b/modules/media/gtkgstsink.c @@ -25,9 +25,20 @@ #include "gtkgstpaintableprivate.h" #include "gtkintl.h" +#if GST_GL_HAVE_WINDOW_X11 && GST_GL_HAVE_PLATFORM_GLX && defined (GDK_WINDOWING_X11) +#include <gdk/x11/gdkx.h> +#include <gst/gl/x11/gstgldisplay_x11.h> +#endif + +#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (GDK_WINDOWING_WAYLAND) +#include <gdk/wayland/gdkwayland.h> +#include <gst/gl/wayland/gstgldisplay_wayland.h> +#endif + enum { PROP_0, PROP_PAINTABLE, + PROP_GL_CONTEXT, N_PROPS, }; @@ -37,11 +48,14 @@ GST_DEBUG_CATEGORY (gtk_debug_gst_sink); #define FORMATS "{ BGRA, ARGB, RGBA, ABGR, RGB, BGR }" +#define NOGL_CAPS GST_VIDEO_CAPS_MAKE (FORMATS) +#define GL_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") + static GstStaticPadTemplate gtk_gst_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) + GST_STATIC_CAPS (GL_CAPS "; " NOGL_CAPS) ); G_DEFINE_TYPE_WITH_CODE (GtkGstSink, gtk_gst_sink, @@ -53,34 +67,74 @@ static GParamSpec *properties[N_PROPS] = { NULL, }; static void -gtk_gst_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, - GstClockTime * start, GstClockTime * end) +gtk_gst_sink_get_times (GstBaseSink *bsink, + GstBuffer *buf, + GstClockTime *start, + GstClockTime *end) { GtkGstSink *gtk_sink; gtk_sink = GTK_GST_SINK (bsink); - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - *start = GST_BUFFER_TIMESTAMP (buf); - if (GST_BUFFER_DURATION_IS_VALID (buf)) - *end = *start + GST_BUFFER_DURATION (buf); - else { - if (GST_VIDEO_INFO_FPS_N (>k_sink->v_info) > 0) { - *end = *start + - gst_util_uint64_scale_int (GST_SECOND, - GST_VIDEO_INFO_FPS_D (>k_sink->v_info), - GST_VIDEO_INFO_FPS_N (>k_sink->v_info)); - } + if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) + { + *start = GST_BUFFER_TIMESTAMP (buf); + if (GST_BUFFER_DURATION_IS_VALID (buf)) + { + *end = *start + GST_BUFFER_DURATION (buf); + } + else + { + if (GST_VIDEO_INFO_FPS_N (>k_sink->v_info) > 0) + *end = *start + gst_util_uint64_scale_int (GST_SECOND, + GST_VIDEO_INFO_FPS_D (>k_sink->v_info), + GST_VIDEO_INFO_FPS_N (>k_sink->v_info)); + } } - } +} + +static GstCaps * +gtk_gst_sink_get_caps (GstBaseSink *bsink, + GstCaps *filter) +{ + GtkGstSink *self = GTK_GST_SINK (bsink); + GstCaps *tmp; + GstCaps *result; + + if (self->gst_context) + { + tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink)); + } + else + { + tmp = gst_caps_from_string (NOGL_CAPS); + } + GST_DEBUG_OBJECT (self, "advertising own caps %" GST_PTR_FORMAT, tmp); + + if (filter) + { + GST_DEBUG_OBJECT (self, "intersecting with filter caps %" GST_PTR_FORMAT, filter); + + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } + else + { + result = tmp; + } + + GST_DEBUG_OBJECT (self, "returning caps: %" GST_PTR_FORMAT, result); + + return result; } static gboolean -gtk_gst_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) +gtk_gst_sink_set_caps (GstBaseSink *bsink, + GstCaps *caps) { GtkGstSink *self = GTK_GST_SINK (bsink); - GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); + GST_DEBUG_OBJECT (self, "set caps with %" GST_PTR_FORMAT, caps); if (!gst_video_info_from_caps (&self->v_info, caps)) return FALSE; @@ -88,6 +142,83 @@ gtk_gst_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) return TRUE; } +static gboolean +gtk_gst_sink_query (GstBaseSink *bsink, + GstQuery *query) +{ + GtkGstSink *self = GTK_GST_SINK (bsink); + + if (GST_QUERY_TYPE (query) == GST_QUERY_CONTEXT && + self->gst_display != NULL && + gst_gl_handle_context_query (GST_ELEMENT (self), query, self->gst_display, self->gst_context, self->gst_app_context)) + return TRUE; + + return GST_BASE_SINK_CLASS (gtk_gst_sink_parent_class)->query (bsink, query); +} + +static gboolean +gtk_gst_sink_propose_allocation (GstBaseSink *bsink, + GstQuery *query) +{ + GtkGstSink *self = GTK_GST_SINK (bsink); + GstBufferPool *pool = NULL; + GstStructure *config; + GstCaps *caps; + guint size; + gboolean need_pool; + + if (!self->gst_context) + return FALSE; + + gst_query_parse_allocation (query, &caps, &need_pool); + + if (caps == NULL) + { + GST_DEBUG_OBJECT (bsink, "no caps specified"); + return FALSE; + } + + if (!gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) + return FALSE; + + if (need_pool) + { + GstVideoInfo info; + + if (!gst_video_info_from_caps (&info, caps)) + { + GST_DEBUG_OBJECT (self, "invalid caps specified"); + return FALSE; + } + + GST_DEBUG_OBJECT (self, "create new pool"); + pool = gst_gl_buffer_pool_new (self->gst_context); + + /* the normal size of a frame */ + size = info.size; + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, config)) + { + GST_DEBUG_OBJECT (bsink, "failed setting config"); + gst_object_unref (pool); + return FALSE; + } + + /* we need at least 2 buffer because we hold on to the last one */ + gst_query_add_allocation_pool (query, pool, size, 2, 0); + gst_object_unref (pool); + } + + /* we also support various metadata */ + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + + return TRUE; +} + + static GdkMemoryFormat gtk_gst_memory_format_from_video (GstVideoFormat format) { @@ -117,21 +248,38 @@ gtk_gst_sink_texture_from_buffer (GtkGstSink *self, { GstVideoFrame frame; GdkTexture *texture; - GBytes *bytes; - - if (!gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ)) - return NULL; - bytes = g_bytes_new_with_free_func (frame.data[0], - frame.info.width * frame.info.stride[0], - (GDestroyNotify) gst_buffer_unref, - gst_buffer_ref (buffer)); - texture = gdk_memory_texture_new (frame.info.width, + if (buffer->pool && GST_IS_GL_BUFFER_POOL (buffer->pool) && + gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ | GST_MAP_GL)) + { + texture = gdk_gl_texture_new (self->gdk_context, + *(guint *) frame.data[0], + frame.info.width, frame.info.height, - gtk_gst_memory_format_from_video (GST_VIDEO_FRAME_FORMAT (&frame)), - bytes, - frame.info.stride[0]); - gst_video_frame_unmap (&frame); + (GDestroyNotify) gst_buffer_unref, + gst_buffer_ref (buffer)); + gst_video_frame_unmap (&frame); + } + else if (gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ)) + { + GBytes *bytes; + + bytes = g_bytes_new_with_free_func (frame.data[0], + frame.info.width * frame.info.stride[0], + (GDestroyNotify) gst_buffer_unref, + gst_buffer_ref (buffer)); + texture = gdk_memory_texture_new (frame.info.width, + frame.info.height, + gtk_gst_memory_format_from_video (GST_VIDEO_FRAME_FORMAT (&frame)), + bytes, + frame.info.stride[0]); + gst_video_frame_unmap (&frame); + } + else + { + GST_ERROR_OBJECT (self, "Could not convert buffer to texture."); + texture = NULL; + } return texture; } @@ -161,6 +309,98 @@ gtk_gst_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) } static void +gtk_gst_sink_initialize_gl (GtkGstSink *self) +{ + GdkDisplay *display; + GError *error = NULL; + + display = gdk_gl_context_get_display (self->gdk_context); + + gdk_gl_context_make_current (self->gdk_context); + +#if GST_GL_HAVE_WINDOW_X11 && GST_GL_HAVE_PLATFORM_GLX && defined (GDK_WINDOWING_X11) + if (GDK_IS_X11_DISPLAY (display)) + { + GstGLPlatform platform = GST_GL_PLATFORM_GLX; + GstGLAPI gl_api; + guintptr gl_handle; + + GST_DEBUG_OBJECT (self, "got GLX on X11!"); + + gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL); + gl_handle = gst_gl_context_get_current_gl_context (platform); + if (gl_handle) + { + self->gst_display = GST_GL_DISPLAY (gst_gl_display_x11_new_with_display (gdk_x11_display_get_xdisplay (display))); + self->gst_app_context = gst_gl_context_new_wrapped (self->gst_display, gl_handle, platform, gl_api); + } + else + { + GST_ERROR_OBJECT (self, "Failed to get handle from GdkGLContext, not using GLX"); + return; + } + } + else +#endif +#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (GDK_WINDOWING_WAYLAND) + if (GDK_IS_WAYLAND_DISPLAY (display)) + { + GstGLPlatform platform = GST_GL_PLATFORM_GLX; + GstGLAPI gl_api; + guintptr gl_handle; + + GST_DEBUG_OBJECT (self, "got EGL on Wayland!"); + + platform = GST_GL_PLATFORM_EGL; + gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL); + gl_handle = gst_gl_context_get_current_gl_context (platform); + + if (gl_handle) + { + struct wl_display *wayland_display; + + wayland_display = gdk_wayland_display_get_wl_display (display); + self->gst_display = GST_GL_DISPLAY (gst_gl_display_wayland_new_with_display (wayland_display)); + self->gst_app_context = gst_gl_context_new_wrapped (self->gst_display, gl_handle, platform, gl_api); + } + else + { + GST_ERROR_OBJECT (self, "Failed to get handle from GdkGLContext, not using Wayland EGL"); + return; + } + } + else +#endif + { + GST_INFO_OBJECT (self, "Unsupported GDK display %s for GL", G_OBJECT_TYPE_NAME (display)); + return; + } + + g_assert (self->gst_app_context != NULL); + + gst_gl_context_activate (self->gst_app_context, TRUE); + if (!gst_gl_context_fill_info (self->gst_app_context, &error)) + { + GST_ERROR_OBJECT (self, "failed to retrieve GDK context info: %s", error->message); + g_clear_error (&error); + g_clear_object (&self->gst_app_context); + g_clear_object (&self->gst_display); + return; + } else { + gst_gl_context_activate (self->gst_app_context, FALSE); + } + + if (!gst_gl_display_create_context (self->gst_display, self->gst_app_context, &self->gst_context, &error)) + { + GST_ERROR_OBJECT (self, "Couldn't create GL context: %s", error->message); + g_error_free (error); + g_clear_object (&self->gst_app_context); + g_clear_object (&self->gst_display); + return; + } +} + +static void gtk_gst_sink_set_property (GObject *object, guint prop_id, const GValue *value, @@ -177,6 +417,12 @@ gtk_gst_sink_set_property (GObject *object, self->paintable = GTK_GST_PAINTABLE (gtk_gst_paintable_new ()); break; + case PROP_GL_CONTEXT: + self->gdk_context = g_value_dup_object (value); + if (self->gdk_context != NULL) + gtk_gst_sink_initialize_gl (self); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -208,6 +454,9 @@ gtk_gst_sink_dispose (GObject *object) { GtkGstSink *self = GTK_GST_SINK (object); + g_clear_object (&self->gst_app_context); + g_clear_object (&self->gst_display); + g_clear_object (&self->gdk_context); g_clear_object (&self->paintable); G_OBJECT_CLASS (gtk_gst_sink_parent_class)->dispose (object); @@ -227,6 +476,9 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass) gstbasesink_class->set_caps = gtk_gst_sink_set_caps; gstbasesink_class->get_times = gtk_gst_sink_get_times; + gstbasesink_class->query = gtk_gst_sink_query; + gstbasesink_class->propose_allocation = gtk_gst_sink_propose_allocation; + gstbasesink_class->get_caps = gtk_gst_sink_get_caps; gstvideosink_class->show_frame = gtk_gst_sink_show_frame; @@ -242,6 +494,18 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass) GTK_TYPE_GST_PAINTABLE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS); + /** + * GtkGstSink:gl-context: + * + * The #GdkGLContext to use for GL rendering. + */ + properties[PROP_GL_CONTEXT] = + g_param_spec_object ("gl-context", + P_("gl-context"), + P_("GL context to use for rendering"), + GDK_TYPE_GL_CONTEXT, + G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, N_PROPS, properties); gst_element_class_set_metadata (gstelement_class, diff --git a/modules/media/gtkgstsinkprivate.h b/modules/media/gtkgstsinkprivate.h index ceb3aa5207..b96fea989f 100644 --- a/modules/media/gtkgstsinkprivate.h +++ b/modules/media/gtkgstsinkprivate.h @@ -24,6 +24,8 @@ #include "gtkgstpaintableprivate.h" #include <gst/gst.h> +#define GST_USE_UNSTABLE_API +#include <gst/gl/gl.h> #include <gst/video/gstvideosink.h> #include <gst/video/video.h> @@ -47,6 +49,11 @@ struct _GtkGstSink GstVideoInfo v_info; GtkGstPaintable * paintable; + + GdkGLContext * gdk_context; + GstGLDisplay * gst_display; + GstGLContext * gst_app_context; + GstGLContext * gst_context; }; struct _GtkGstSinkClass diff --git a/modules/media/meson.build b/modules/media/meson.build index 3268824c3e..424de06e7e 100644 --- a/modules/media/meson.build +++ b/modules/media/meson.build @@ -20,7 +20,7 @@ if gstreamer_enabled c_args: [ '-DGTK_COMPILATION' ], - dependencies: [ libgtk_dep, gstplayer_dep ], + dependencies: [ libgtk_dep, gstreamer_deps ], install_dir: media_install_dir, install : true) endif |