codecparsers: av1: add valid check for global motion params.

The global motion params and its matrix values need to be verified
before we use them. If it is invalid, we should notify the decoder
that it should not be used.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1464>

2 files changed, 162 insertions, 0 deletions

diff --git a/gst-libs/gst/codecparsers/gstav1parser.c b/gst-libs/gst/codecparsers/gstav1parser.c
index 3812d230a..92a3307e1 100644
--- a/gst-libs/gst/codecparsers/gstav1parser.c
+++ b/gst-libs/gst/codecparsers/gstav1parser.c
@@ -195,6 +195,88 @@ av1_helper_inverse_recenter (gint r, gint v)
     return r + (v >> 1);
 }
 
+/* Shift down with rounding for use when n >= 0, value >= 0 */
+static guint64
+av1_helper_round_power_of_two (guint64 value, guint16 n)
+{
+  return (value + (((guint64) (1) << n) >> 1)) >> n;
+}
+
+ /* Shift down with rounding for signed integers, for use when n >= 0 */
+static gint64
+av1_helper_round_power_of_two_signed (gint64 value, guint16 n)
+{
+  return (value < 0) ? -((gint64) (av1_helper_round_power_of_two (-value, n)))
+      : (gint64) av1_helper_round_power_of_two (value, n);
+}
+
+static gint
+av1_helper_msb (guint n)
+{
+  int log = 0;
+  guint value = n;
+  int i;
+
+  g_assert (n != 0);
+
+  for (i = 4; i >= 0; --i) {
+    const gint shift = (1 << i);
+    const guint x = value >> shift;
+    if (x != 0) {
+      value = x;
+      log += shift;
+    }
+  }
+  return log;
+}
+
+static const guint16 div_lut[GST_AV1_DIV_LUT_NUM + 1] = {
+  16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768,
+  15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142,
+  15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564,
+  14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028,
+  13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530,
+  13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066,
+  13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633,
+  12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228,
+  12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848,
+  11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491,
+  11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155,
+  11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838,
+  10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538,
+  10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255,
+  10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986,
+  9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732,
+  9709, 9687, 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489,
+  9468, 9447, 9425, 9404, 9383, 9362, 9341, 9321, 9300, 9279, 9259,
+  9239, 9218, 9198, 9178, 9158, 9138, 9118, 9098, 9079, 9059, 9039,
+  9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, 8867, 8849, 8830,
+  8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, 8630,
+  8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439,
+  8422, 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257,
+  8240, 8224, 8208, 8192,
+};
+
+static gint16
+av1_helper_resolve_divisor_32 (guint32 D, gint16 * shift)
+{
+  gint32 f;
+  gint32 e;
+
+  *shift = av1_helper_msb (D);
+  // e is obtained from D after resetting the most significant 1 bit.
+  e = D - ((guint32) 1 << *shift);
+  // Get the most significant DIV_LUT_BITS (8) bits of e into f
+  if (*shift > GST_AV1_DIV_LUT_BITS)
+    f = av1_helper_round_power_of_two (e, *shift - GST_AV1_DIV_LUT_BITS);
+  else
+    f = e << (GST_AV1_DIV_LUT_BITS - *shift);
+  g_assert (f <= GST_AV1_DIV_LUT_NUM);
+  *shift += GST_AV1_DIV_LUT_PREC_BITS;
+  // Use f as lookup into the precomputed table of multipliers
+  return div_lut[f];
+}
+
 /*************************************
  *                                   *
  * Bitstream Functions               *
@@ -2798,6 +2880,66 @@ gst_av1_parse_global_param (GstAV1Parser * parser,
   return GST_AV1_PARSER_OK;
 }
 
+static gboolean
+gst_av1_parser_is_shear_params_valid (gint32 gm_params[6])
+{
+  const gint32 *mat = gm_params;
+  gint16 alpha, beta, gamma, delta;
+  gint16 shift;
+  gint16 y;
+  gint16 v;
+  guint i;
+  gboolean default_warp_params;
+
+  if (!(mat[2] > 0))
+    return FALSE;
+
+  default_warp_params = TRUE;
+  for (i = 0; i < 6; i++) {
+    if (gm_params[i] != ((i % 3 == 2) ? 1 << GST_AV1_WARPEDMODEL_PREC_BITS : 0)) {
+      default_warp_params = FALSE;
+      break;
+    }
+  }
+  if (default_warp_params)
+    return TRUE;
+
+  alpha = CLAMP (mat[2] - (1 << GST_AV1_WARPEDMODEL_PREC_BITS),
+      G_MININT16, G_MAXINT16);
+  beta = CLAMP (mat[3], G_MININT16, G_MAXINT16);
+  y = av1_helper_resolve_divisor_32 (ABS (mat[2]), &shift)
+      * (mat[2] < 0 ? -1 : 1);
+  v = ((gint64) mat[4] * (1 << GST_AV1_WARPEDMODEL_PREC_BITS)) * y;
+  gamma =
+      CLAMP ((gint) av1_helper_round_power_of_two_signed (v, shift), G_MININT16,
+      G_MAXINT16);
+  v = ((gint64) mat[3] * mat[4]) * y;
+  delta =
+      CLAMP (mat[5] - (gint) av1_helper_round_power_of_two_signed (v,
+          shift) - (1 << GST_AV1_WARPEDMODEL_PREC_BITS), G_MININT16,
+      G_MAXINT16);
+
+  alpha =
+      av1_helper_round_power_of_two_signed (alpha,
+      GST_AV1_WARP_PARAM_REDUCE_BITS) * (1 << GST_AV1_WARP_PARAM_REDUCE_BITS);
+  beta =
+      av1_helper_round_power_of_two_signed (beta,
+      GST_AV1_WARP_PARAM_REDUCE_BITS) * (1 << GST_AV1_WARP_PARAM_REDUCE_BITS);
+  gamma =
+      av1_helper_round_power_of_two_signed (gamma,
+      GST_AV1_WARP_PARAM_REDUCE_BITS) * (1 << GST_AV1_WARP_PARAM_REDUCE_BITS);
+  delta =
+      av1_helper_round_power_of_two_signed (delta,
+      GST_AV1_WARP_PARAM_REDUCE_BITS) * (1 << GST_AV1_WARP_PARAM_REDUCE_BITS);
+
+  if ((4 * ABS (alpha) + 7 * ABS (beta) >= (1 << GST_AV1_WARPEDMODEL_PREC_BITS))
+      || (4 * ABS (gamma) + 4 * ABS (delta) >=
+          (1 << GST_AV1_WARPEDMODEL_PREC_BITS)))
+    return FALSE;
+
+  return TRUE;
+}
+
 /* 5.9.24 */
 static GstAV1ParserResult
 gst_av1_parse_global_motion_params (GstAV1Parser * parser,
@@ -2812,6 +2954,7 @@ gst_av1_parse_global_motion_params (GstAV1Parser * parser,
   /* init value */
   gm_params->gm_type[GST_AV1_REF_INTRA_FRAME] = GST_AV1_WARP_MODEL_IDENTITY;
   for (ref = GST_AV1_REF_LAST_FRAME; ref <= GST_AV1_REF_ALTREF_FRAME; ref++) {
+    gm_params->invalid[ref] = 0;
     gm_params->gm_type[ref] = GST_AV1_WARP_MODEL_IDENTITY;
     for (i = 0; i < 6; i++) {
       gm_params->gm_params[ref][i] =
@@ -2903,6 +3046,10 @@ gst_av1_parse_global_motion_params (GstAV1Parser * parser,
       if (retval != GST_AV1_PARSER_OK)
         goto error;
     }
+
+    if (type <= GST_AV1_WARP_MODEL_AFFINE)
+      gm_params->invalid[ref] =
+          !gst_av1_parser_is_shear_params_valid (gm_params->gm_params[ref]);
   }
 
 success:
diff --git a/gst-libs/gst/codecparsers/gstav1parser.h b/gst-libs/gst/codecparsers/gstav1parser.h
index 7a6152e11..cda392fd5 100644
--- a/gst-libs/gst/codecparsers/gstav1parser.h
+++ b/gst-libs/gst/codecparsers/gstav1parser.h
@@ -59,6 +59,7 @@ G_BEGIN_DECLS
 #define GST_AV1_GM_TRANS_PREC_BITS             6
 #define GST_AV1_GM_TRANS_ONLY_PREC_BITS        3
 #define GST_AV1_WARPEDMODEL_PREC_BITS          16
+#define GST_AV1_WARP_PARAM_REDUCE_BITS         6
 #define GST_AV1_SELECT_SCREEN_CONTENT_TOOLS    2
 #define GST_AV1_SELECT_INTEGER_MV              2
 #define GST_AV1_RESTORATION_TILESIZE_MAX       256
@@ -79,6 +80,11 @@ G_BEGIN_DECLS
 #define GST_AV1_MAX_NUM_POS_LUMA               25
 #define GST_AV1_MAX_NUM_PLANES                 3
 
+#define GST_AV1_DIV_LUT_PREC_BITS              14
+#define GST_AV1_DIV_LUT_BITS                   8
+#define GST_AV1_DIV_LUT_NUM                    (1 << GST_AV1_DIV_LUT_BITS)
+
+
 typedef struct _GstAV1Parser GstAV1Parser;
 
 typedef struct _GstAV1OBUHeader GstAV1OBUHeader;
@@ -1257,6 +1263,14 @@ struct _GstAV1LoopRestorationParams {
  * @gm_params: is set equal to SavedGmParams[ frame_to_show_map_idx ][ ref ][ j ] for
  *   ref = LAST_FRAME..ALTREF_FRAME, for j = 0..5.
  * @gm_type: specifying the type of global motion.
+ * @invalid: whether this global motion parameters is invalid. (Since: 1.20)
+ */
+/**
+ * _GstAV1GlobalMotionParams.invalid:
+ *
+ * whether this global motion parameters is invalid.
+ *
+ * Since: 1.20
  */
 struct _GstAV1GlobalMotionParams {
   gboolean is_global[GST_AV1_NUM_REF_FRAMES];
@@ -1265,6 +1279,7 @@ struct _GstAV1GlobalMotionParams {
   gint32 gm_params[GST_AV1_NUM_REF_FRAMES][6];
 
   GstAV1WarpModelType gm_type[GST_AV1_NUM_REF_FRAMES]; /* GmType */
+  gboolean invalid[GST_AV1_NUM_REF_FRAMES];
 };
 
 /**