pdfwrite - detect duplicate images with SMask entries

Bug #706418 "Detect duplicate images with SMask"

We were not adding the SMask entry to the image dictionary until after
we had checked to see if the image (stream and dictionary) was identical
to a prior instance. The prior instance, of course, was written and
hashed with the SMask entry present, so the two instances never matched.

Fix by adding the SMask entry before we complete the image and check for
duplicates.

This exposed a bug in smask_image_check. The pointer to a character
being tested was never incremented, leading to an infinite loop, Clearly
this code was never previously being used.

2 files changed, 25 insertions, 14 deletions

diff --git a/devices/vector/gdevpdfi.c b/devices/vector/gdevpdfi.c
index e4681de58..ac8bb99d9 100644
--- a/devices/vector/gdevpdfi.c
+++ b/devices/vector/gdevpdfi.c
@@ -1912,8 +1912,29 @@ static int
 pdf_end_and_do_image(gx_device_pdf *pdev, pdf_image_writer *piw,
                      const gs_matrix *mat, gs_id ps_bitmap_id, pdf_image_usage_t do_image)
 {
-    int code = pdf_end_write_image(pdev, piw);
-    pdf_resource_t *pres = piw->pres;
+    int code = 0;
+    pdf_resource_t *pres = NULL;
+
+    /* In order to identify duplicated images which use the same SMask we must
+     * add the SMask entry to the image dictionary before we call pdf_end_write_image
+     * because that will hash the dictionaries and streams. If we haven't already
+     * added the SMask then the hash will not match any existing image which uses an SMask.
+     */
+    if (do_image == USE_AS_IMAGE) {
+        if (pdev->image_mask_id != gs_no_id && piw->pres && piw->pres->object) {
+            char buf[20];
+
+            gs_snprintf(buf, sizeof(buf), "%ld 0 R", pdev->image_mask_id);
+            code = cos_dict_put_string_copy((cos_dict_t *)piw->pres->object,
+                    pdev->image_mask_is_SMask ? "/SMask" : "/Mask", buf);
+            (*(piw->pres->object)).md5_valid = 0;
+            if (code < 0)
+                return code;
+        }
+    }
+
+    code = pdf_end_write_image(pdev, piw);
+    pres = piw->pres;
 
     switch (code) {
     default:
@@ -1923,16 +1944,6 @@ pdf_end_and_do_image(gx_device_pdf *pdev, pdf_image_writer *piw,
         break;
     case 0:
         if (do_image == USE_AS_IMAGE) {
-            if (pdev->image_mask_id != gs_no_id) {
-                char buf[20];
-
-                gs_snprintf(buf, sizeof(buf), "%ld 0 R", pdev->image_mask_id);
-                code = cos_dict_put_string_copy((cos_dict_t *)pres->object,
-                        pdev->image_mask_is_SMask ? "/SMask" : "/Mask", buf);
-                (*(pres->object)).md5_valid = 0;
-                if (code < 0)
-                    return code;
-            }
             if (pdev->image_mask_skip)
                 code = 0;
             else
diff --git a/devices/vector/gdevpdfj.c b/devices/vector/gdevpdfj.c
index 5888ac705..4ab58bc2f 100644
--- a/devices/vector/gdevpdfj.c
+++ b/devices/vector/gdevpdfj.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2001-2022 Artifex Software, Inc.
+/* Copyright (C) 2001-2023 Artifex Software, Inc.
    All Rights Reserved.
 
    This software is provided AS-IS with no warranty, either express or
@@ -524,7 +524,7 @@ smask_image_check(gx_device_pdf * pdev, pdf_resource_t *pres0, pdf_resource_t *p
                 if (p > v->contents.chars.data + v->contents.chars.size)
                     return 0;
                 ix *= 10;
-                ix += (*p) - 0x30;
+                ix += (*p++) - 0x30;
             }
             if (ix != pdev->image_mask_id)
                 return 0;