diff options
| -rw-r--r-- | configure.ac | 19 | ||||
| -rw-r--r-- | docs/plugins/Makefile.am | 1 | ||||
| -rw-r--r-- | docs/plugins/gst-plugins-bad-plugins-docs.sgml | 1 | ||||
| -rw-r--r-- | docs/plugins/gst-plugins-bad-plugins-sections.txt | 12 | ||||
| -rw-r--r-- | ext/Makefile.am | 7 | ||||
| -rw-r--r-- | ext/iqa/Makefile.am | 27 | ||||
| -rw-r--r-- | ext/iqa/iqa.c | 343 | ||||
| -rw-r--r-- | ext/iqa/iqa.h | 64 |
8 files changed, 474 insertions, 0 deletions
diff --git a/configure.ac b/configure.ac index c48b86cda..35ef31f8a 100644 --- a/configure.ac +++ b/configure.ac @@ -358,6 +358,23 @@ AC_SUBST(EXIF_LIBS) AC_SUBST(EXIF_CFLAGS) AM_CONDITIONAL(USE_EXIF, test "x$HAVE_EXIF" = "xyes") +AG_GST_CHECK_FEATURE(IQA, [iqa], iqa , [ + PKG_CHECK_MODULES(DSSIM, dssim, [ + HAVE_DSSIM="yes" + HAVE_IQA="yes" + ], [ + HAVE_DSSIM="no" + HAVE_IQA="no" + ]) + + AM_CONDITIONAL(HAVE_DSSIM, test "x$HAVE_DSSIM" = "xyes") + if test "x$HAVE_DSSIM" = "xyes"; then + AC_DEFINE(HAVE_DSSIM, 1, [Define if you have dssim library]) + fi + AC_SUBST(DSSIM_LIBS) + AC_SUBST(DSSIM_CFLAGS) +]) + dnl Orc ORC_CHECK([0.4.17]) @@ -3525,6 +3542,7 @@ AM_CONDITIONAL(USE_GSM, false) AM_CONDITIONAL(USE_GTK3, false) AM_CONDITIONAL(USE_GTK3_GL, false) AM_CONDITIONAL(USE_HLS, false) +AM_CONDITIONAL(USE_IQA, false) AM_CONDITIONAL(USE_KATE, false) AM_CONDITIONAL(USE_KMS, false) AM_CONDITIONAL(USE_TIGER, false) @@ -3846,6 +3864,7 @@ ext/flite/Makefile ext/fluidsynth/Makefile ext/gsm/Makefile ext/hls/Makefile +ext/iqa/Makefile ext/kate/Makefile ext/ladspa/Makefile ext/lv2/Makefile diff --git a/docs/plugins/Makefile.am b/docs/plugins/Makefile.am index 0dc14f88f..0cce8594a 100644 --- a/docs/plugins/Makefile.am +++ b/docs/plugins/Makefile.am @@ -75,6 +75,7 @@ EXTRA_HFILES = \ $(top_srcdir)/ext/dts/gstdtsdec.h \ $(top_srcdir)/ext/faac/gstfaac.h \ $(top_srcdir)/ext/faad/gstfaad.h \ + $(top_srcdir)/ext/iqa/iqa.h \ $(top_srcdir)/ext/kate/gstkateenc.h \ $(top_srcdir)/ext/kate/gstkatedec.h \ $(top_srcdir)/ext/kate/gstkateparse.h \ diff --git a/docs/plugins/gst-plugins-bad-plugins-docs.sgml b/docs/plugins/gst-plugins-bad-plugins-docs.sgml index 70b1ea017..199d66fb6 100644 --- a/docs/plugins/gst-plugins-bad-plugins-docs.sgml +++ b/docs/plugins/gst-plugins-bad-plugins-docs.sgml @@ -104,6 +104,7 @@ <xi:include href="xml/element-glvideomixerelement.xml" /> <xi:include href="xml/element-glvideomixer.xml" /> <xi:include href="xml/element-glviewconvert.xml" /> + <xi:include href="xml/element-iqa.xml" /> <xi:include href="xml/element-jpegparse.xml" /> <xi:include href="xml/element-kaleidoscope.xml" /> <xi:include href="xml/element-liveadder.xml" /> diff --git a/docs/plugins/gst-plugins-bad-plugins-sections.txt b/docs/plugins/gst-plugins-bad-plugins-sections.txt index 811c0e5f0..9596f4389 100644 --- a/docs/plugins/gst-plugins-bad-plugins-sections.txt +++ b/docs/plugins/gst-plugins-bad-plugins-sections.txt @@ -2282,6 +2282,18 @@ gst_interlace_get_type </SECTION> <SECTION> +<FILE>element-iqa</FILE> +<TITLE>IQA</TITLE> +Iqa +<SUBSECTION Standard> +IqaClass +IQA +GST_TYPE_IQA +iqa_get_type +gst_iqa_plugin_init +</SECTION> + +<SECTION> <FILE>element-ivfparse</FILE> <TITLE>ivfparse</TITLE> GstIvfParse diff --git a/ext/Makefile.am b/ext/Makefile.am index 16821e0be..6dfc6bc5a 100644 --- a/ext/Makefile.am +++ b/ext/Makefile.am @@ -142,6 +142,12 @@ else GSM_DIR= endif +if USE_IQA +IQA_DIR = iqa +else +IQA_DIR = +endif + if USE_KATE KATE_DIR=kate else @@ -457,6 +463,7 @@ SUBDIRS=\ $(FLUIDSYNTH_DIR) \ $(GSM_DIR) \ $(G729_DIR) \ + $(IQA_DIR) \ $(KATE_DIR) \ $(LADSPA_DIR) \ $(LV2_DIR) \ diff --git a/ext/iqa/Makefile.am b/ext/iqa/Makefile.am new file mode 100644 index 000000000..f114d8d09 --- /dev/null +++ b/ext/iqa/Makefile.am @@ -0,0 +1,27 @@ +plugin_LTLIBRARIES = libgstiqa.la + +libgstiqa_la_SOURCES = \ + iqa.c + +libgstiqa_la_CFLAGS = \ + -I$(top_srcdir)/gst-libs \ + -I$(top_builddir)/gst-libs \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(GST_BASE_CFLAGS) $(GST_CFLAGS) + +libgstiqa_la_CFLAGS += $(DSSIM_CFLAGS) + +libgstiqa_la_LIBADD = \ + $(top_builddir)/gst-libs/gst/base/libgstbadbase-$(GST_API_VERSION).la \ + $(top_builddir)/gst-libs/gst/video/libgstbadvideo-$(GST_API_VERSION).la \ + $(GST_PLUGINS_BASE_LIBS) \ + $(GST_BASE_LIBS) $(GST_LIBS) + +libgstiqa_la_LIBADD += $(DSSIM_LIBS) + +libgstiqa_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstiqa_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + iqa.h + diff --git a/ext/iqa/iqa.c b/ext/iqa/iqa.c new file mode 100644 index 000000000..5ba60b35d --- /dev/null +++ b/ext/iqa/iqa.c @@ -0,0 +1,343 @@ +/* Image Quality Assessment plugin + * Copyright (C) 2015 Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-iqa + * @short_description: Image Quality Assessment plugin. + * + * IQA will perform full reference image quality assessment, with the + * first added pad being the reference. + * + * It will perform comparisons on video streams with the same geometry. + * + * The image output will be the heat map of differences, between + * the two pads with the highest measured difference. + * + * For each reference frame, IQA will post a message containing + * a structure named IQA. + * + * The only metric supported for now is "dssim", which will be available + * if https://github.com/pornel/dssim was installed on the system + * at the time that plugin was compiled. + * + * For each metric activated, this structure will contain another + * structure, named after the metric. + * + * The message will also contain a "time" field. + * + * For example, if do-dssim is set to true, and there are + * two compared streams, the emitted structure will look like this: + * + * IQA, dssim=(structure)"dssim\,\ sink_1\=\(double\)0.053621271267184856\,\ + * sink_2\=\(double\)0.0082939683976297474\;", + * time=(guint64)0; + * + * <refsect2> + * <title>Example launch line</title> + * |[ + * gst-launch-1.0 -m uridecodebin uri=file:///test/file/1 ! iqa name=iqa do-dssim=true \ + * ! videoconvert ! autovideosink uridecodebin uri=file:///test/file/2 ! iqa. + * ]| This pipeline will output messages to the console for each set of compared frames. + * </refsect2> + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "iqa.h" + +#ifdef HAVE_DSSIM +#include "dssim.h" +#endif + +GST_DEBUG_CATEGORY_STATIC (gst_iqa_debug); +#define GST_CAT_DEFAULT gst_iqa_debug + +#define SINK_FORMATS " { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\ + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\ + " RGBx, BGRx } " + +#define SRC_FORMAT " { RGBA } " + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SRC_FORMAT)) + ); + +enum +{ + PROP_0, + PROP_DO_SSIM, + PROP_LAST, +}; + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SINK_FORMATS)) + ); + + +/* GstIqa */ + +#define gst_iqa_parent_class parent_class +G_DEFINE_TYPE (GstIqa, gst_iqa, GST_TYPE_VIDEO_AGGREGATOR); + +#ifdef HAVE_DSSIM + +inline static unsigned char +to_byte (float in) +{ + if (in <= 0) + return 0; + if (in >= 255.f / 256.f) + return 255; + return in * 256.f; +} + +static void +do_dssim (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp, + GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname) +{ + dssim_attr *attr = dssim_create_attr (); + gint y; + unsigned char **ptrs, **ptrs2; + GstMapInfo ref_info; + GstMapInfo cmp_info; + GstMapInfo out_info; + dssim_image *ref_image; + dssim_image *cmp_image; + double dssim; + dssim_ssim_map map_meta; + float *map; + gint i; + dssim_rgba *out; + GstStructure *dssim_structure; + + gst_structure_get (msg_structure, "dssim", GST_TYPE_STRUCTURE, + &dssim_structure, NULL); + + dssim_set_save_ssim_maps (attr, 1, 1); + if (ref->info.width != cmp->info.width || + ref->info.height != cmp->info.height) { + GST_WARNING_OBJECT (self, + "Cannot compare two images with a different geometry yet"); + return; + } + + gst_buffer_map (ref->buffer, &ref_info, GST_MAP_READ); + gst_buffer_map (cmp->buffer, &cmp_info, GST_MAP_READ); + gst_buffer_map (outbuf, &out_info, GST_MAP_WRITE); + out = (dssim_rgba *) out_info.data; + + ptrs = g_malloc (sizeof (char **) * ref->info.height); + ptrs2 = g_malloc (sizeof (char **) * cmp->info.height); + + for (y = 0; y < ref->info.height; y++) { + ptrs[y] = ref_info.data + (ref->info.width * 4 * y); + } + + ref_image = + dssim_create_image (attr, ptrs, DSSIM_RGBA, ref->info.width, + ref->info.height, 0.45455); + + ptrs2 = g_malloc (sizeof (char **) * cmp->info.height); + + for (y = 0; y < cmp->info.height; y++) { + ptrs2[y] = cmp_info.data + (cmp->info.width * 4 * y); + } + + cmp_image = + dssim_create_image (attr, ptrs2, DSSIM_RGBA, cmp->info.width, + cmp->info.height, 0.45455); + dssim = dssim_compare (attr, ref_image, cmp_image); + + map_meta = dssim_pop_ssim_map (attr, 0, 0); + + if (dssim > self->max_dssim) { + map = map_meta.data; + + for (i = 0; i < map_meta.width * map_meta.height; i++) { + const float max = 1.0 - map[i]; + const float maxsq = max * max; + out[i] = (dssim_rgba) { + .r = to_byte (max * 3.0),.g = to_byte (maxsq * 6.0),.b = + to_byte (max / ((1.0 - map_meta.dssim) * 4.0)),.a = 255,}; + } + self->max_dssim = dssim; + } + + gst_structure_set (dssim_structure, padname, G_TYPE_DOUBLE, dssim, NULL); + gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE, + dssim_structure, NULL); + gst_structure_free (dssim_structure); + + g_free (ptrs); + g_free (ptrs2); + gst_buffer_unmap (ref->buffer, &ref_info); + gst_buffer_unmap (cmp->buffer, &cmp_info); + gst_buffer_unmap (outbuf, &out_info); + dssim_dealloc_image (ref_image); + dssim_dealloc_image (cmp_image); + dssim_dealloc_attr (attr); +} +#else +static void +do_dssim (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp, + GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname) +{ +} +#endif + +static void +compare_frames (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp, + GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname) +{ + if (self->do_dssim) + do_dssim (self, ref, cmp, outbuf, msg_structure, padname); +} + +static GstFlowReturn +gst_iqa_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) +{ + GList *l; + GstVideoFrame *ref_frame = NULL; + GstIqa *self = GST_IQA (vagg); + GstStructure *msg_structure = gst_structure_new_empty ("IQA"); + GstMessage *m = gst_message_new_element (GST_OBJECT (self), msg_structure); + GstAggregator *agg = GST_AGGREGATOR (vagg); + + if (self->do_dssim) { + gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE, + gst_structure_new_empty ("dssim"), NULL); + self->max_dssim = 0.0; + } + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + + if (pad->aggregated_frame != NULL) { + if (!ref_frame) { + ref_frame = pad->aggregated_frame; + } else { + gchar *padname = gst_pad_get_name (pad); + GstVideoFrame *cmp_frame = pad->aggregated_frame; + + compare_frames (self, ref_frame, cmp_frame, outbuf, msg_structure, + padname); + g_free (padname); + } + } + } + + GST_OBJECT_UNLOCK (vagg); + + /* We only post the message here, because we can't post it while the object + * is locked. + */ + gst_structure_set (msg_structure, "time", GST_TYPE_CLOCK_TIME, + agg->segment.position, NULL); + gst_element_post_message (GST_ELEMENT (self), m); + return GST_FLOW_OK; +} + +static void +_set_property (GObject * object, guint prop_id, const GValue * value, + GParamSpec * pspec) +{ + GstIqa *self = GST_IQA (object); + + switch (prop_id) { + case PROP_DO_SSIM: + self->do_dssim = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstIqa *self = GST_IQA (object); + + switch (prop_id) { + case PROP_DO_SSIM: + g_value_set_boolean (value, self->do_dssim); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* GObject boilerplate */ +static void +gst_iqa_class_init (GstIqaClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *gstelement_class = (GstElementClass *) klass; + GstVideoAggregatorClass *videoaggregator_class = + (GstVideoAggregatorClass *) klass; + + videoaggregator_class->aggregate_frames = gst_iqa_aggregate_frames; + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&sink_factory)); + + gobject_class->set_property = _set_property; + gobject_class->get_property = _get_property; + +#ifdef HAVE_DSSIM + g_object_class_install_property (gobject_class, PROP_DO_SSIM, + g_param_spec_boolean ("do-dssim", "do-dssim", + "Run structural similarity checks", FALSE, G_PARAM_READWRITE)); +#endif + + gst_element_class_set_static_metadata (gstelement_class, "Iqa", + "Filter/Analyzer/Video", + "Provides various Image Quality Assessment metrics", + "Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>"); +} + +static void +gst_iqa_init (GstIqa * self) +{ +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_iqa_debug, "iqa", 0, "iqa"); + + return gst_element_register (plugin, "iqa", GST_RANK_PRIMARY, GST_TYPE_IQA); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + iqa, + "Iqa", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, + GST_PACKAGE_ORIGIN) diff --git a/ext/iqa/iqa.h b/ext/iqa/iqa.h new file mode 100644 index 000000000..e879daa26 --- /dev/null +++ b/ext/iqa/iqa.h @@ -0,0 +1,64 @@ +/* Image Quality Assessment plugin + * Copyright (C) 2015 Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_IQA_H__ +#define __GST_IQA_H__ + +#include <gst/gst.h> +#include <gst/video/video.h> +#include <gst/video/gstvideoaggregator.h> + +G_BEGIN_DECLS + +#define GST_TYPE_IQA (gst_iqa_get_type()) +#define GST_IQA(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_IQA, GstIqa)) +#define GST_IQA_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_IQA, GstIqaClass)) +#define GST_IS_IQA(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_IQA)) +#define GST_IS_IQA_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_IQA)) + +typedef struct _GstIqa GstIqa; +typedef struct _GstIqaClass GstIqaClass; + +/** + * GstIqa: + * + * The opaque #GstIqa structure. + */ +struct _GstIqa +{ + GstVideoAggregator videoaggregator; + + gboolean do_dssim; + double max_dssim; +}; + +struct _GstIqaClass +{ + GstVideoAggregatorClass parent_class; +}; + +GType gst_iqa_get_type (void); + +G_END_DECLS +#endif /* __GST_IQA_H__ */ + |