gaudi: fix exclusion's factor range

Avoid dividing by zero when the factor is zero. Instead, output a buffer with
all color values as zero.

1 files changed, 9 insertions, 0 deletions

diff --git a/gst/gaudieffects/gstexclusion.c b/gst/gaudieffects/gstexclusion.c
index 8eaacf1f5..60e963282 100644
--- a/gst/gaudieffects/gstexclusion.c
+++ b/gst/gaudieffects/gstexclusion.c
@@ -270,6 +270,9 @@ transform (guint32 * src, guint32 * dest, gint video_area, gint factor)
   guint32 in;
   gint x, red, green, blue;
 
+  if (G_UNLIKELY (factor == 0))
+    return;
+
   for (x = 0; x < video_area; x++) {
     in = *src++;
 
@@ -277,6 +280,12 @@ transform (guint32 * src, guint32 * dest, gint video_area, gint factor)
     green = (in >> 8) & 0xff;
     blue = (in) & 0xff;
 
+    if (factor == 0) {
+      red = 0;
+      green = 0;
+      blue = 0;
+    }
+
     red = factor -
         (((factor - red) * (factor - red) / factor) + ((green * red) / factor));
     green = factor -