GhostPDF, pdfwrite, graphics library - fix WMode usage

OSS-fuzz #58582

The fundamental problem here is that pdfwrite was assuming that the
font WMode could only ever be 0 or 1 (the only two valid values) and so
was using it as a bitfield, shifting and OR'ing it with other values.

The file in this case has a CMap which contains :
/WMode 8883123282518010140455180910294889 def

Which gets clamped to the maximum unsigned integer 0x7fffff

This led to a non-zero value in the flags to the glyph info code, when
the value *should* have been 0, which caused the graphics library to
take a code path which wasn't valid. This led to us trying to use a
member of a structure whose pointer was NULL.

I can't be certain whether other places in the code use WMode in the
same way, so I've chosen to fix this at several levels.

Firstly, in the code path we shouldn't reach (gs_type42_glyph_info_by_gid)
check the value of pmat before calling gs_default_glyph_info. That code
will try to use the matrix to scale the outline, so if it is NULL then
the result is undefined. This prevents the seg fault.

Secondly, in gdevpdtc.c, scan_cmap_text(), set wmode to be either 0 or
1, to ensure that it does work as a bit, rather than using the integer
value from the font and assuming it will be 0 or 1.

Finally in the three places in the PDF interpreter where we set the
WMode for the font, check to see if the value is either 0 or 1 and if it
is not, raise a warning and make it 0 or 1.

6 files changed, 26 insertions, 4 deletions

diff --git a/base/gstype42.c b/base/gstype42.c
index 4525744ae..b6f106820 100644
--- a/base/gstype42.c
+++ b/base/gstype42.c
@@ -1385,6 +1385,9 @@ gs_type42_glyph_info_by_gid(gs_font *font, gs_glyph glyph, const gs_matrix *pmat
 
     outline.memory = pfont->memory;
     if (default_members) {
+        if (pmat == NULL)
+            return gs_note_error(gs_error_undefinedresult);
+
         code = gs_default_glyph_info(font, glyph, pmat, default_members, info);
 
         if (code < 0)
diff --git a/devices/vector/gdevpdtc.c b/devices/vector/gdevpdtc.c
index eb366ce10..ba4ea78cb 100644
--- a/devices/vector/gdevpdtc.c
+++ b/devices/vector/gdevpdtc.c
@@ -431,7 +431,7 @@ scan_cmap_text(pdf_text_enum_t *pte, void *vbuf)
     gs_font_type0 *const font = (gs_font_type0 *)pte->orig_font; /* Type 0, fmap_CMap */
     /* Not sure. Changed for CDevProc callout. Was pte->current_font */
     gs_text_enum_t scan = *(gs_text_enum_t *)pte;
-    int wmode = font->WMode, code, rcode = 0;
+    int wmode = font->WMode == 0 ? 0 : 1, code, rcode = 0;
     pdf_font_resource_t *pdsubf0 = NULL;
     gs_font *subfont0 = NULL, *saved_subfont = NULL;
     uint index = scan.index, xy_index = scan.xy_index, start_index = index;
diff --git a/pdf/pdf_cmap.c b/pdf/pdf_cmap.c
index 18cbbc078..676a61110 100644
--- a/pdf/pdf_cmap.c
+++ b/pdf/pdf_cmap.c
@@ -630,7 +630,13 @@ static int cmap_def_func(gs_memory_t *mem, pdf_ps_ctx_t *s, byte *buf, byte *buf
         }
         else if (!memcmp(s->cur[-1].val.name, CMAP_NAME_AND_LEN("WMode"))) {
             if (pdf_ps_obj_has_type(&s->cur[0], PDF_PS_OBJ_INTEGER)) {
-                pdficmap->wmode = s->cur[0].val.i;
+                if (s->cur[0].val.i != 0) {
+                    if (s->cur[0].val.i != 1)
+                        pdfi_set_warning(s->pdfi_ctx, 0, NULL, W_PDF_BAD_WMODE, "cmap_def_func", NULL);
+                    pdficmap->wmode = 1;
+                }
+                else
+                    pdficmap->wmode = 0;
             }
             else {
                 pdficmap->wmode = 0;
diff --git a/pdf/pdf_font1.c b/pdf/pdf_font1.c
index 7fc37fe40..8e3d0e3b7 100644
--- a/pdf/pdf_font1.c
+++ b/pdf/pdf_font1.c
@@ -569,7 +569,13 @@ pdfi_read_type1_font(pdf_context *ctx, pdf_dict *font_dict, pdf_dict *stream_dic
             if (fpriv.gsu.gst1.UID.id != 0)
                 memcpy(&pfont1->UID, &fpriv.gsu.gst1.UID, sizeof(pfont1->UID));
             fpriv.gsu.gst1.UID.xvalues = NULL; /* In case of error */
-            pfont1->WMode = fpriv.gsu.gst1.WMode;
+            if (fpriv.gsu.gst1.WMode != 0) {
+                if (fpriv.gsu.gst1.WMode != 1)
+                    pdfi_set_warning(ctx, 0, NULL, W_PDF_BAD_WMODE, "pdfi_read_type1_font", NULL);
+                pfont1->WMode = 1;
+            }
+            else
+                pfont1->WMode = 0;
             pfont1->PaintType = fpriv.gsu.gst1.PaintType;
             pfont1->StrokeWidth = fpriv.gsu.gst1.StrokeWidth;
 
diff --git a/pdf/pdf_fontps.c b/pdf/pdf_fontps.c
index 95201c5ac..6a46069d6 100644
--- a/pdf/pdf_fontps.c
+++ b/pdf/pdf_fontps.c
@@ -448,7 +448,13 @@ ps_font_def_func(gs_memory_t *mem, pdf_ps_ctx_t *s, byte *buf, byte *bufend)
               }
               else if (pdf_ps_name_cmp(&s->cur[-1], "WMode")) {
                   if (pdf_ps_obj_has_type(&s->cur[0], PDF_PS_OBJ_INTEGER)) {
-                      priv->gsu.gst1.WMode = s->cur[0].val.i;
+                      if (s->cur[0].val.i != 0) {
+                          if (s->cur[0].val.i != 1)
+                            pdfi_set_warning(s->pdfi_ctx, 0, NULL, W_PDF_BAD_WMODE, "ps_font_def_func", NULL);
+                        priv->gsu.gst1.WMode = 1;
+                      }
+                      else
+                        priv->gsu.gst1.WMode = 0;
                   }
               }
               else if (pdf_ps_name_cmp(&s->cur[-1], "lenIV")) {
diff --git a/pdf/pdf_warnings.h b/pdf/pdf_warnings.h
index 0d3fca4f0..6402d8f13 100644
--- a/pdf/pdf_warnings.h
+++ b/pdf/pdf_warnings.h
@@ -96,4 +96,5 @@ PARAM(W_PDF_UNBLANACED_BT,          "A page ended after a BT had been executed a
 PARAM(W_PDF_MISMATCH_GENERATION,    "The generation number of an indirectly referenced object did not match the xref"),
 PARAM(W_PDF_BAD_RENDERINGINTENT,    "A ri or /RI used an unknown named rendering intent"),
 PARAM(W_PDF_BAD_VIEW,               "Couldn't read the initial document view"),
+PARAM(W_PDF_BAD_WMODE,              "A Font or CMap has a WMode which is neither 0 (horizontal) nor 1 (vertical)"),
 #undef PARAM