GhostPDF - fix some transparency blending problems

The initial fix here is from Michael who has done all the diagnostic
work on this.

The original code was always pushing an Isolated group, if it pushed
a group. This was incorrect.

Michael then pointed out that pdfi_shading_setup_trans()
does a gsave/grestore around pdfi_trans_setup(). But that function
can potentially change the blending mode (and push a group) and if
we grestore after it, then we throw that away.

The reason we do the gsave/grestore is because we must create a path
so that we can give a bbox to the group. If we have a BBox in the
Shading we can just use that. If we don't then we need to use the
current clip. In order to use the current clip we call clippath() to
create a path from it in the graphics state.

Obviously this changes the graphics state, which is something we don't
want to happen, so we gsave/grestore round it. There is similar code in
pdfi_image_setup_trans().

To avoid this, we no longer use the original pdfi_trans_setup() which
was in any case a one line wrapper around the code doing the real work.

Instead we always call the old 'inner' routine, which means we must
always either supply a bbox, or ensure that the current path is valid
and supply NULL for a bbox.

For stroke and fill the current path is fine. For shadings and images
we now gsave, construct a path, get its bbox, grestore and then pass
the bbox to the new pdfi_trans_setup() function (which is the old
'inner' function renamed). This avoids doing a gsave/grestore round
the potential change in blending mode.

5 files changed, 62 insertions, 32 deletions

diff --git a/pdf/pdf_image.c b/pdf/pdf_image.c
index b62e8d703..9c56e4bdd 100644
--- a/pdf/pdf_image.c
+++ b/pdf/pdf_image.c
@@ -1039,19 +1039,36 @@ static int
 pdfi_image_setup_trans(pdf_context *ctx, pdfi_trans_state_t *state)
 {
     int code;
+    gs_rect bbox;
 
+    /* We need to create a bbox in order to pass it to the transparency setup,
+     * which (potentially, at least, uses it to set up a transparency group.
+     * Setting up a 1x1 path, and establishing it's BBox will work, because
+     * the image scaling is already in place. We don't want to disturb the
+     * graphics state, so do this inside a gsave/grestore pair.
+     */
     code = pdfi_gsave(ctx);
     if (code < 0)
         return code;
+
+    code = gs_newpath(ctx->pgs);
+    if (code < 0)
+        goto exit;
     code = gs_moveto(ctx->pgs, 1.0, 1.0);
     if (code < 0)
         goto exit;
     code = gs_lineto(ctx->pgs, 0., 0.);
     if (code < 0)
         goto exit;
-    code = pdfi_trans_setup(ctx, state, TRANSPARENCY_Caller_Image);
- exit:
+
+    code = pdfi_get_current_bbox(ctx, &bbox, false);
+    if (code < 0)
+        goto exit;
+
     pdfi_grestore(ctx);
+
+    code = pdfi_trans_setup(ctx, state, &bbox, TRANSPARENCY_Caller_Image);
+ exit:
     return code;
 }
 
diff --git a/pdf/pdf_path.c b/pdf/pdf_path.c
index a116501a4..ef6f1328d 100644
--- a/pdf/pdf_path.c
+++ b/pdf/pdf_path.c
@@ -123,7 +123,7 @@ static int pdfi_fill_inner(pdf_context *ctx, bool use_eofill)
     code = pdfi_gsave(ctx);
     if (code < 0) goto exit;
 
-    code = pdfi_trans_setup(ctx, &state, TRANSPARENCY_Caller_Fill);
+    code = pdfi_trans_setup(ctx, &state, NULL, TRANSPARENCY_Caller_Fill);
     if (code == 0) {
         if (use_eofill)
             code = gs_eofill(ctx->pgs);
@@ -168,7 +168,7 @@ int pdfi_stroke(pdf_context *ctx)
     if (code < 0) goto exit;
 
     gs_swapcolors_quick(ctx->pgs);
-    code = pdfi_trans_setup(ctx, &state, TRANSPARENCY_Caller_Stroke);
+    code = pdfi_trans_setup(ctx, &state, NULL, TRANSPARENCY_Caller_Stroke);
     if (code == 0) {
         code = gs_stroke(ctx->pgs);
         code1 = pdfi_trans_teardown(ctx, &state);
@@ -381,7 +381,7 @@ static int pdfi_B_inner(pdf_context *ctx, bool use_eofill)
     code = pdfi_gsave(ctx);
     if (code < 0) goto exit;
 
-    code = pdfi_trans_setup(ctx, &state, TRANSPARENCY_Caller_FillStroke);
+    code = pdfi_trans_setup(ctx, &state, NULL, TRANSPARENCY_Caller_FillStroke);
     if (code == 0) {
         if (use_eofill)
             code = gs_eofillstroke(ctx->pgs, &code1);
diff --git a/pdf/pdf_shading.c b/pdf/pdf_shading.c
index f4caf2b9d..eb985296d 100644
--- a/pdf/pdf_shading.c
+++ b/pdf/pdf_shading.c
@@ -788,43 +788,58 @@ static int
 pdfi_shading_setup_trans(pdf_context *ctx, pdfi_trans_state_t *state, pdf_obj *Shading)
 {
     int code;
-    gs_rect bbox;
+    gs_rect bbox, *box = NULL;
     pdf_array *BBox = NULL;
     pdf_dict *shading_dict;
 
-    code = pdfi_gsave(ctx);
-    if (code < 0)
-        return code;
-
     code = pdfi_dict_from_obj(ctx, Shading, &shading_dict);
     if (code < 0)
         return code;
 
     code = pdfi_dict_knownget_type(ctx, shading_dict, "BBox", PDF_ARRAY, (pdf_obj **)&BBox);
+    if (code < 0)
+        goto exit;
+
     if (code > 0) {
         code = pdfi_array_to_gs_rect(ctx, BBox, &bbox);
+        if (code >= 0)
+            box = &bbox;
+    }
+
+    /* If we didn't get a BBox for the shading, then we need to create one, in order to
+     * pass it to the transparency setup, which (potentially, at least, uses it to set
+     * up a transparency group.
+     * In the basence of anything better, we take the currnet clip, turn that into a path
+     * and then get the bounding box of that path. Obviously we don't want to disturb the
+     * current path in the graphics state, so we do a gsave/grestore round it.
+     */
+    if (box == NULL) {
+        code = pdfi_gsave(ctx);
         if (code < 0)
             goto exit;
-        code = gs_moveto(ctx->pgs, bbox.p.x, bbox.p.y);
+
+        code = gs_newpath(ctx->pgs);
         if (code < 0)
-            goto exit;
-        code = gs_lineto(ctx->pgs, bbox.q.x, 0.);
+            goto bbox_error;
+
+        code = gs_clippath(ctx->pgs);
         if (code < 0)
-            goto exit;
-        code = gs_lineto(ctx->pgs, 0., bbox.q.y);
+            goto bbox_error;
+
+        code = pdfi_get_current_bbox(ctx, &bbox, false);
+
+bbox_error:
+        pdfi_grestore(ctx);
+
         if (code < 0)
             goto exit;
-        code = gs_closepath(ctx->pgs);
-    } else {
-        code = gs_clippath(ctx->pgs);
+
+        box = &bbox;
     }
-    if (code < 0)
-        goto exit;
+    code = pdfi_trans_setup(ctx, state, box, TRANSPARENCY_Caller_Other);
 
-    code = pdfi_trans_setup(ctx, state, TRANSPARENCY_Caller_Other);
  exit:
     pdfi_countdown(BBox);
-    pdfi_grestore(ctx);
     return code;
 }
 
diff --git a/pdf/pdf_trans.c b/pdf/pdf_trans.c
index 367c8be29..12047c325 100644
--- a/pdf/pdf_trans.c
+++ b/pdf/pdf_trans.c
@@ -654,7 +654,7 @@ static bool pdfi_trans_okOPcs(pdf_context *ctx)
     return false;
 }
 
-static int pdfi_trans_setup_inner(pdf_context *ctx, pdfi_trans_state_t *state, gs_rect *bbox,
+int pdfi_trans_setup(pdf_context *ctx, pdfi_trans_state_t *state, gs_rect *bbox,
                            pdfi_transparency_caller_t caller)
 {
     pdfi_int_gstate *igs = (pdfi_int_gstate *)ctx->pgs->client_data;
@@ -708,7 +708,9 @@ static int pdfi_trans_setup_inner(pdf_context *ctx, pdfi_trans_state_t *state, g
     /* TODO: error handling... */
     if (need_group) {
         stroked_bbox = (caller == TRANSPARENCY_Caller_Stroke || caller == TRANSPARENCY_Caller_FillStroke);
-        code = pdfi_trans_begin_simple_group(ctx, bbox, stroked_bbox, true, false);
+        /* When changing to compatible overprint bm, the group pushed must be non-isolated. The exception
+           is if we have a softmask.  See /setupOPtrans in pdf_ops.ps */
+        code = pdfi_trans_begin_simple_group(ctx, bbox, stroked_bbox, igs->SMask != NULL, false);
         state->GroupPushed = true;
         state->saveStrokeAlpha = gs_getstrokeconstantalpha(ctx->pgs);
         state->saveFillAlpha = gs_getfillconstantalpha(ctx->pgs);
@@ -750,11 +752,11 @@ int pdfi_trans_setup_text(pdf_context *ctx, pdfi_trans_state_t *state, bool is_s
 
     switch (Trmode) {
     case 0:
-        code = pdfi_trans_setup_inner(ctx, state, &bbox, TRANSPARENCY_Caller_Fill);
+        code = pdfi_trans_setup(ctx, state, &bbox, TRANSPARENCY_Caller_Fill);
         break;
     default:
         /* TODO: All the others */
-        code = pdfi_trans_setup_inner(ctx, state, &bbox, TRANSPARENCY_Caller_Fill);
+        code = pdfi_trans_setup(ctx, state, &bbox, TRANSPARENCY_Caller_Fill);
         break;
     }
 
@@ -771,11 +773,6 @@ int pdfi_trans_teardown_text(pdf_context *ctx, pdfi_trans_state_t *state)
     return code;
 }
 
-int pdfi_trans_setup(pdf_context *ctx, pdfi_trans_state_t *state, pdfi_transparency_caller_t caller)
-{
-    return pdfi_trans_setup_inner(ctx, state, NULL, caller);
-}
-
 int pdfi_trans_teardown(pdf_context *ctx, pdfi_trans_state_t *state)
 {
     int code = 0;
diff --git a/pdf/pdf_trans.h b/pdf/pdf_trans.h
index f1bb89dcb..e2ac28b3a 100644
--- a/pdf/pdf_trans.h
+++ b/pdf/pdf_trans.h
@@ -35,7 +35,8 @@ typedef enum {
 
 int pdfi_trans_setup_text(pdf_context *ctx, pdfi_trans_state_t *state, bool is_show);
 int pdfi_trans_teardown_text(pdf_context *ctx, pdfi_trans_state_t *state);
-int pdfi_trans_setup(pdf_context *ctx, pdfi_trans_state_t *state, pdfi_transparency_caller_t caller);
+int pdfi_trans_setup(pdf_context *ctx, pdfi_trans_state_t *state, gs_rect *bbox, pdfi_transparency_caller_t caller);
+
 int pdfi_trans_teardown(pdf_context *ctx, pdfi_trans_state_t *state);
 
 int pdfi_trans_begin_simple_group(pdf_context *ctx, gs_rect *bbox, bool stroked_bbox, bool isolated, bool knockout);