GhostPDF - SMask with Matte needs the image colour space for the group

When we encounter an image with an SMask we must push an Isolated group
to render the image. If the SMask image has a /Matte entry, then we need
to push the group with the colour space of the image.

Since the image code is the only client for the transparency routine
pdfi_trans_begin_isolated_group() it's simples to make that routine take
an additional parameter which is the colour space to be used for the
group, or NULL if none.

We can then directly use the passed in colour space to set the params
ColorSpace member for the group.

To facilitate determining whether an image SMask has a Matte or not,
its convenient to modify pdfi_image_get_matte() to return a boolean
indicating that. This was in fact already being used for that purpose
(pdfi_image_setup_type3x) by checking the return value was positive,
but I think it's more obvious to have the function update a specific
boolean parameter.

This is propagated back through pdfi_do_image_smask() for the benefit
of pdfi_do_image().

This fixes 4 files:
tests_private/pdf/PDF_1.7_FTS/fts_26_2605.pdf
tests_private/pdf/uploads/bug700571.pdf

These 2 files only exhibited a problem with psddcmyk16 for some reason.
tests_private/pdf/sumatra/uninitialized_value_with_JPX_images.pdf
tests_private/pdf/PDF_1.7_FTS/fts_17_1718.pdf

3 files changed, 19 insertions, 12 deletions

diff --git a/pdf/pdf_image.c b/pdf/pdf_image.c
index 9c56e4bdd..a52e5e8e0 100644
--- a/pdf/pdf_image.c
+++ b/pdf/pdf_image.c
@@ -912,7 +912,7 @@ pdfi_data_image_params(pdf_context *ctx, pdfi_image_info_t *info,
 
 /* Returns number of components in Matte array or 0 if not found, <0 if error */
 static int
-pdfi_image_get_matte(pdf_context *ctx, pdf_obj *smask_obj, float *vals, int size)
+pdfi_image_get_matte(pdf_context *ctx, pdf_obj *smask_obj, float *vals, int size, bool *has_Matte)
 {
     int i;
     pdf_array *Matte = NULL;
@@ -920,6 +920,7 @@ pdfi_image_get_matte(pdf_context *ctx, pdf_obj *smask_obj, float *vals, int size
     double f;
     pdf_dict *smask_dict = NULL;
 
+    *has_Matte = false;
     code = pdfi_dict_from_obj(ctx, smask_obj, &smask_dict);
     if (code < 0)
         goto exit;
@@ -929,6 +930,7 @@ pdfi_image_get_matte(pdf_context *ctx, pdf_obj *smask_obj, float *vals, int size
     if (code <= 0)
         goto exit;
 
+    *has_Matte = true;
     if (pdfi_array_size(Matte) > size) {
         code = gs_note_error(gs_error_rangecheck);
         goto exit;
@@ -950,7 +952,7 @@ pdfi_image_get_matte(pdf_context *ctx, pdf_obj *smask_obj, float *vals, int size
 
 /* See ztrans.c/zbegintransparencymaskimage() and pdf_draw.ps/doimagesmask */
 static int
-pdfi_do_image_smask(pdf_context *ctx, pdf_c_stream *source, pdfi_image_info_t *image_info)
+pdfi_do_image_smask(pdf_context *ctx, pdf_c_stream *source, pdfi_image_info_t *image_info, bool *has_Matte)
 {
     gs_rect bbox = { { 0, 0} , { 1, 1} };
     gs_transparency_mask_params_t params;
@@ -977,7 +979,7 @@ pdfi_do_image_smask(pdf_context *ctx, pdf_c_stream *source, pdfi_image_info_t *i
 
     gs_trans_mask_params_init(&params, TRANSPARENCY_MASK_Luminosity);
 
-    code = pdfi_image_get_matte(ctx, image_info->SMask, params.Matte, GS_CLIENT_COLOR_MAX_COMPONENTS);
+    code = pdfi_image_get_matte(ctx, image_info->SMask, params.Matte, GS_CLIENT_COLOR_MAX_COMPONENTS, has_Matte);
 
     if (code >= 0)
         params.Matte_components = code;
@@ -1178,9 +1180,9 @@ pdfi_image_setup_type3x(pdf_context *ctx, pdfi_image_info_t *image_info,
         mask = &t3ximage->Opacity;
     mask->InterleaveType = 3;
 
-    code = pdfi_image_get_matte(ctx, image_info->SMask, mask->Matte, GS_CLIENT_COLOR_MAX_COMPONENTS);
-    if (code > 0)
-        mask->has_Matte = true;
+    code = pdfi_image_get_matte(ctx, image_info->SMask, mask->Matte, GS_CLIENT_COLOR_MAX_COMPONENTS, &mask->has_Matte);
+    if (code < 0)
+        return code;
 
     code = pdfi_data_image_params(ctx, smask_info, &mask->MaskDict, comps, NULL);
     return code;
@@ -1752,21 +1754,26 @@ pdfi_do_image(pdf_context *ctx, pdf_dict *page_dict, pdf_dict *stream_dict, pdf_
     }
 
     if (ctx->page.has_transparency == true && image_info.SMask != NULL) {
+        bool has_Matte = false;
+
         /* If this flag is set, then device will process the SMask and we need do nothing
          * here (e.g. pdfwrite).
          */
         if (!ctx->device_state.preserve_smask) {
-            code = pdfi_do_image_smask(ctx, source, &image_info);
+            code = pdfi_do_image_smask(ctx, source, &image_info, &has_Matte);
             if (code < 0)
                 goto cleanupExit;
             need_smask_cleanup = true;
         }
-        code = pdfi_trans_begin_isolated_group(ctx, true);
+        if (has_Matte)
+            code = pdfi_trans_begin_isolated_group(ctx, true, pcs);
+        else
+            code = pdfi_trans_begin_isolated_group(ctx, true, NULL);
         if (code < 0)
             goto cleanupExit;
         transparency_group = true;
     } else if (igs->SMask) {
-        code = pdfi_trans_begin_isolated_group(ctx, false);
+        code = pdfi_trans_begin_isolated_group(ctx, false, NULL);
         if (code < 0)
             goto cleanupExit;
         transparency_group = true;
diff --git a/pdf/pdf_trans.c b/pdf/pdf_trans.c
index 12047c325..93ab8a6c0 100644
--- a/pdf/pdf_trans.c
+++ b/pdf/pdf_trans.c
@@ -526,14 +526,14 @@ int pdfi_trans_end_simple_group(pdf_context *ctx)
 }
 
 
-int pdfi_trans_begin_isolated_group(pdf_context *ctx, bool image_with_SMask)
+int pdfi_trans_begin_isolated_group(pdf_context *ctx, bool image_with_SMask, gs_color_space *pcs)
 {
     gs_transparency_group_params_t params;
     gs_rect bbox;
 
     gs_trans_group_params_init(&params, 1.0);
 
-    params.ColorSpace = NULL;
+    params.ColorSpace = pcs;
     params.Isolated = true;
     params.Knockout = false;
     params.image_with_SMask = image_with_SMask;
diff --git a/pdf/pdf_trans.h b/pdf/pdf_trans.h
index e2ac28b3a..e0057b744 100644
--- a/pdf/pdf_trans.h
+++ b/pdf/pdf_trans.h
@@ -45,7 +45,7 @@ int pdfi_trans_begin_form_group(pdf_context *ctx, pdf_dict *page_dict, pdf_dict
 int pdfi_trans_end_group(pdf_context *ctx);
 int pdfi_trans_end_simple_group(pdf_context *ctx);
 int pdfi_trans_set_params(pdf_context *ctx);
-int pdfi_trans_begin_isolated_group(pdf_context *ctx, bool image_with_SMask);
+int pdfi_trans_begin_isolated_group(pdf_context *ctx, bool image_with_SMask, gs_color_space *pcs);
 int pdfi_trans_end_isolated_group(pdf_context *ctx);
 int pdfi_trans_end_smask_notify(pdf_context *ctx);
 void pdfi_trans_set_needs_OP(pdf_context *ctx);