New base function `FT_Matrix_Check' (#54019).

* src/base/ftcalc.c (FT_Matrix_Check): New base function to properly
reject degenerate font matrices.

* include/freetype/internal/ftcalc.h: Updated.

* src/cff/cffparse.c (cff_parse_font_matrix), src/cid/cidload.c
(cid_parse_font_matrix), src/type1/t1load.c (t1_parse_font_matrix),
src/type42/t42parse.c (t42_parse_font_matrix): Use
`FT_Matrix_Check'.

7 files changed, 140 insertions, 17 deletions

diff --git a/ChangeLog b/ChangeLog
index 06df64ead..0234e60e8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2018-06-24  Werner Lemberg  <wl@gnu.org>
+
+	New base function `FT_Matrix_Check' (#54019).
+
+	* src/base/ftcalc.c (FT_Matrix_Check): New base function to properly
+	reject degenerate font matrices.
+
+	* include/freetype/internal/ftcalc.h: Updated.
+
+	* src/cff/cffparse.c (cff_parse_font_matrix), src/cid/cidload.c
+	(cid_parse_font_matrix), src/type1/t1load.c (t1_parse_font_matrix),
+	src/type42/t42parse.c (t42_parse_font_matrix): Use
+	`FT_Matrix_Check'.
+
 2018-06-23  Werner Lemberg  <wl@gnu.org>
 
 	Fix typo.
diff --git a/include/freetype/internal/ftcalc.h b/include/freetype/internal/ftcalc.h
index a04a3cb80..38c6f7741 100644
--- a/include/freetype/internal/ftcalc.h
+++ b/include/freetype/internal/ftcalc.h
@@ -293,6 +293,21 @@ FT_BEGIN_HEADER
 
 
   /*
+   * Check a matrix.  If the transformation would lead to extreme shear or
+   * extreme scaling, for example, return 0.  If everything is OK, return 1.
+   *
+   * Based on geometric considerations we use the following inequality to
+   * identify a degenerate matrix.
+   *
+   *   50 * abs(xx*yy - xy*yx) < xx^2 + xy^2 + yx^2 + yy^2
+   *
+   * Value 50 is heuristic.
+   */
+  FT_BASE( FT_Bool )
+  FT_Matrix_Check( const FT_Matrix*  matrix );
+
+
+  /*
    * A variant of FT_Vector_Transform.  See comments for
    * FT_Matrix_Multiply_Scaled.
    */
diff --git a/src/base/ftcalc.c b/src/base/ftcalc.c
index 6abc4edc0..c96d5d2d6 100644
--- a/src/base/ftcalc.c
+++ b/src/base/ftcalc.c
@@ -747,6 +747,72 @@
 
   /* documentation is in ftcalc.h */
 
+  FT_BASE_DEF( FT_Bool )
+  FT_Matrix_Check( const FT_Matrix*  matrix )
+  {
+    FT_Matrix  m;
+    FT_Fixed   val[4];
+    FT_Fixed   nonzero_minval, maxval;
+    FT_Fixed   temp1, temp2;
+    FT_UInt    i;
+
+
+    if ( !matrix )
+      return 0;
+
+    val[0] = FT_ABS( matrix->xx );
+    val[1] = FT_ABS( matrix->xy );
+    val[2] = FT_ABS( matrix->yx );
+    val[3] = FT_ABS( matrix->yy );
+
+    /*
+     * To avoid overflow, we ensure that each value is not larger than
+     *
+     *   int(sqrt(2^31 / 4)) = 23170  ;
+     *
+     * we also check that no value becomes zero if we have to scale.
+     */
+
+    maxval         = 0;
+    nonzero_minval = FT_LONG_MAX;
+
+    for ( i = 0; i < 4; i++ )
+    {
+      if ( val[i] > maxval )
+        maxval = val[i];
+      if ( val[i] && val[i] < nonzero_minval )
+        nonzero_minval = val[i];
+    }
+
+    if ( maxval > 23170 )
+    {
+      FT_Fixed  scale = FT_DivFix( maxval, 23170 );
+
+
+      if ( !FT_DivFix( nonzero_minval, scale ) )
+        return 0;    /* value range too large */
+
+      m.xx = FT_DivFix( matrix->xx, scale );
+      m.xy = FT_DivFix( matrix->xy, scale );
+      m.yx = FT_DivFix( matrix->yx, scale );
+      m.yy = FT_DivFix( matrix->yy, scale );
+    }
+    else
+      m = *matrix;
+
+    temp1 = FT_ABS( m.xx * m.yy - m.xy * m.yx );
+    temp2 = m.xx * m.xx + m.xy * m.xy + m.yx * m.yx + m.yy * m.yy;
+
+    if ( temp1 == 0         ||
+         temp2 / temp1 > 50 )
+      return 0;
+
+    return 1;
+  }
+
+
+  /* documentation is in ftcalc.h */
+
   FT_BASE_DEF( void )
   FT_Vector_Transform_Scaled( FT_Vector*        vector,
                               const FT_Matrix*  matrix,
diff --git a/src/cff/cffparse.c b/src/cff/cffparse.c
index 41cc1dde6..0e5f9512b 100644
--- a/src/cff/cffparse.c
+++ b/src/cff/cffparse.c
@@ -604,7 +604,6 @@
     FT_Vector*       offset = &dict->font_offset;
     FT_ULong*        upm    = &dict->units_per_em;
     FT_Byte**        data   = parser->stack;
-    FT_Error         error  = FT_ERR( Stack_Underflow );
 
 
     if ( parser->top >= parser->stack + 6 )
@@ -616,8 +615,6 @@
       int      i;
 
 
-      error = FT_Err_Ok;
-
       dict->has_font_matrix = TRUE;
 
       /* We expect a well-formed font matrix, this is, the matrix elements */
@@ -646,22 +643,11 @@
            ( max_scaling - min_scaling ) < 0 ||
            ( max_scaling - min_scaling ) > 9 )
       {
-        /* Return default matrix in case of unlikely values. */
-
         FT_TRACE1(( "cff_parse_font_matrix:"
                     " strange scaling values (minimum %d, maximum %d),\n"
                     "                      "
                     " using default matrix\n", min_scaling, max_scaling ));
-
-        matrix->xx = 0x10000L;
-        matrix->yx = 0;
-        matrix->xy = 0;
-        matrix->yy = 0x10000L;
-        offset->x  = 0;
-        offset->y  = 0;
-        *upm       = 1;
-
-        goto Exit;
+        goto Unlikely;
       }
 
       for ( i = 0; i < 6; i++ )
@@ -708,10 +694,31 @@
                   (double)matrix->yy / *upm / 65536,
                   (double)offset->x  / *upm / 65536,
                   (double)offset->y  / *upm / 65536 ));
+
+      if ( !FT_Matrix_Check( matrix ) )
+      {
+        FT_TRACE1(( "cff_parse_font_matrix:"
+                    " degenerate values, using default matrix\n" ));
+        goto Unlikely;
+      }
+
+      return FT_Err_Ok;
     }
+    else
+      return FT_THROW( Stack_Underflow );
 
-  Exit:
-    return error;
+  Unlikely:
+    /* Return default matrix in case of unlikely values. */
+
+    matrix->xx = 0x10000L;
+    matrix->yx = 0;
+    matrix->xy = 0;
+    matrix->yy = 0x10000L;
+    offset->x  = 0;
+    offset->y  = 0;
+    *upm       = 1;
+
+    return FT_Err_Ok;
   }
 
 
diff --git a/src/cid/cidload.c b/src/cid/cidload.c
index dc458e8fd..8f2a312ab 100644
--- a/src/cid/cidload.c
+++ b/src/cid/cidload.c
@@ -200,6 +200,13 @@
       matrix->xy = temp[2];
       matrix->yy = temp[3];
 
+      if ( !FT_Matrix_Check( matrix ) )
+      {
+        FT_ERROR(( "t1_parse_font_matrix: invalid font matrix\n" ));
+        parser->root.error = FT_THROW( Invalid_File_Format );
+        return FT_THROW( Invalid_File_Format );
+      }
+
       /* note that the font offsets are expressed in integer font units */
       offset->x  = temp[4] >> 16;
       offset->y  = temp[5] >> 16;
diff --git a/src/type1/t1load.c b/src/type1/t1load.c
index baf4d42b8..8bcf2eaa2 100644
--- a/src/type1/t1load.c
+++ b/src/type1/t1load.c
@@ -1286,6 +1286,13 @@
     matrix->xy = temp[2];
     matrix->yy = temp[3];
 
+    if ( !FT_Matrix_Check( matrix ) )
+    {
+      FT_ERROR(( "t1_parse_font_matrix: invalid font matrix\n" ));
+      parser->root.error = FT_THROW( Invalid_File_Format );
+      return;
+    }
+
     /* note that the offsets must be expressed in integer font units */
     offset->x = temp[4] >> 16;
     offset->y = temp[5] >> 16;
diff --git a/src/type42/t42parse.c b/src/type42/t42parse.c
index 009121fc8..edd27a850 100644
--- a/src/type42/t42parse.c
+++ b/src/type42/t42parse.c
@@ -284,6 +284,13 @@
     matrix->xy = temp[2];
     matrix->yy = temp[3];
 
+    if ( !FT_Matrix_Check( matrix ) )
+    {
+      FT_ERROR(( "t42_parse_font_matrix: invalid font matrix\n" ));
+      parser->root.error = FT_THROW( Invalid_File_Format );
+      return;
+    }
+
     /* note that the offsets must be expressed in integer font units */
     offset->x = temp[4] >> 16;
     offset->y = temp[5] >> 16;