GhostPDF - propagate errors properly

There was a typo in pdfi_trans_set_mask() which meant that if we had
an error from pdfi_form_execgroup() we would overwrite it with the
(probably successful) return code from gs_end_transparency_mask() and
would ignore any error code from that function.

In addition pdfi_trans_set_params didn't pass back any errors from
pdfi_trans_set_mask() leading to them being lost.

Finally, in pdfi_trans_set_mask() if we have already processed a SMask
then don't return 1, return 0 to indicate success.

This series of problems was preventing the gs_error_Fatal code in
gdevp14.c, pdf14_pop_transparency_group() being propagated back to the
caller. Since this is intended to prevent a seg fault on a serious
(should never happen) error condition it's quite important not to lose
it.

1 files changed, 5 insertions, 3 deletions

diff --git a/pdf/pdf_trans.c b/pdf/pdf_trans.c
index d7549745a..fecb74c90 100644
--- a/pdf/pdf_trans.c
+++ b/pdf/pdf_trans.c
@@ -120,6 +120,7 @@ static int pdfi_trans_set_mask(pdf_context *ctx, pdfi_int_gstate *igs, int color
 #if DEBUG_TRANSPARENCY
           dbgmprintf(ctx->memory, "SMask already built, skipping\n");
 #endif
+          code = 0;
           goto exit;
         }
         ProcessedKnown = 1;
@@ -310,7 +311,7 @@ static int pdfi_trans_set_mask(pdf_context *ctx, pdfi_int_gstate *igs, int color
         code = pdfi_form_execgroup(ctx, ctx->page.CurrentPageDict, G_stream,
                                    igs->GroupGState, NULL, &group_Matrix);
         code1 = gs_end_transparency_mask(ctx->pgs, colorindex);
-        if (code != 0)
+        if (code == 0)
             code = code1;
 
         /* Put back the matrix (we couldn't just rely on gsave/grestore for whatever reason,
@@ -878,6 +879,7 @@ int pdfi_trans_teardown(pdf_context *ctx, pdfi_trans_state_t *state)
 
 int pdfi_trans_set_params(pdf_context *ctx)
 {
+    int code = 0;
     pdfi_int_gstate *igs = (pdfi_int_gstate *)ctx->pgs->client_data;
     gs_transparency_channel_selector_t csel;
 
@@ -887,9 +889,9 @@ int pdfi_trans_set_params(pdf_context *ctx)
         else
             csel = TRANSPARENCY_CHANNEL_Opacity;
         if (igs->SMask) {
-            pdfi_trans_set_mask(ctx, igs, csel);
+            code = pdfi_trans_set_mask(ctx, igs, csel);
         }
     }
 
-    return 0;
+    return code;
 }