Fix problems with pngalpha and deep color transparency filling.

The pngalpha device is created with a depth of 32, with 3 color
components; this was confusing pdf14 into thinking that more
than 8 bits per color were being used, and so deep color buffers
were required.

To fix this we update the logic to be smarter; in cases where we
don't have a clear determination based on bits alone, we look
at the max_color and max_gray values. Because this code is now
more complex than before, we pull it into a shared function.

8 files changed, 32 insertions, 64 deletions

diff --git a/base/gdevdflt.c b/base/gdevdflt.c
index 630becf03..f1d1bb886 100644
--- a/base/gdevdflt.c
+++ b/base/gdevdflt.c
@@ -532,10 +532,7 @@ void check_device_compatible_encoding(gx_device *dev)
     gx_color_index mul, color_index;
     int i, j;
     gx_color_value colorants[GX_DEVICE_COLOR_MAX_COMPONENTS];
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (pinfo->separable_and_linear == GX_CINFO_UNKNOWN_SEP_LIN)
         check_device_separable(dev);
diff --git a/base/gdevmem.c b/base/gdevmem.c
index 4f1ab99ae..9a3fe5e06 100644
--- a/base/gdevmem.c
+++ b/base/gdevmem.c
@@ -387,10 +387,7 @@ gdev_mem_max_height(const gx_device_memory * dev, int width, ulong size,
     int height;
     ulong max_height;
     ulong data_size;
-    bool has_tags = device_encodes_tags((gx_device *)dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                          dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (page_uses_transparency) {
         /*
diff --git a/base/gdevp14.c b/base/gdevp14.c
index 18211aabb..d38951a04 100644
--- a/base/gdevp14.c
+++ b/base/gdevp14.c
@@ -3371,10 +3371,7 @@ pdf14_copy_alpha_color(gx_device * dev, const byte * data, int data_x,
                       gx_color_index color, const gx_device_color *pdc,
                       int depth, bool devn)
 {
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (deep)
         return do_pdf14_copy_alpha_color_16(dev, data, data_x, aa_raster,
@@ -4130,10 +4127,7 @@ get_pdf14_device_proto(gx_device * dev, pdf14_device ** pdevproto,
     pdf14_default_colorspace_t dev_cs =
                 pdf14_determine_default_blend_cs(dev, use_pdf14_accum,
                                                  &using_blend_cs);
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     switch (dev_cs) {
         case PDF14_DeviceGray:
@@ -4232,10 +4226,7 @@ pdf14_ok_to_optimize(gx_device *dev)
     int tag_depth = device_encodes_tags(dev) ? 8 : 0;
     cmm_dev_profile_t *dev_profile;
     int code = dev_proc(dev, get_profile)(dev,  &dev_profile);
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (code < 0)
         return false;
@@ -4295,9 +4286,7 @@ pdf14_recreate_device(gs_memory_t *mem,	gs_gstate	* pgs,
     pdf14_device temp_dev_proto;
     bool has_tags = device_encodes_tags(dev);
     int code;
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if_debug0m('v', dev->memory, "[v]pdf14_recreate_device\n");
 
@@ -5215,9 +5204,7 @@ pdf14_update_device_color_procs_push_c(gx_device               *dev,
     cmm_profile_t *icc_profile_dev = NULL;
     gsicc_rendering_param_t render_cond;
     cmm_dev_profile_t *dev_profile;
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    int deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     memset(comp_bits, 0, GX_DEVICE_COLOR_MAX_COMPONENTS);
     memset(comp_shift, 0, GX_DEVICE_COLOR_MAX_COMPONENTS);
@@ -5572,10 +5559,7 @@ pdf14_begin_transparency_mask(gx_device	*dev,
     int code;
     int group_color_numcomps;
     gs_transparency_color_t group_color;
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    int deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (ptmp->subtype == TRANSPARENCY_MASK_None) {
         pdf14_ctx *ctx = pdev->ctx;
@@ -6533,7 +6517,6 @@ gs_pdf14_device_push(gs_memory_t *mem, gs_gstate * pgs,
     uchar k;
     int max_bitmap;
     bool use_pdf14_accum = false;
-    int bits_per_comp;
     bool deep;
 
     /* Guard against later seg faults, this should not be possible */
@@ -6541,9 +6524,7 @@ gs_pdf14_device_push(gs_memory_t *mem, gs_gstate * pgs,
         return gs_throw_code(gs_error_Fatal);
 
     has_tags = device_encodes_tags(target);
-    bits_per_comp = ((target->color_info.depth - has_tags*8) /
-                      target->color_info.num_components);
-    deep = bits_per_comp > 8;
+    deep = device_is_deep(target);
     max_bitmap = target->space_params.MaxBitmap == 0 ? MAX_BITMAP :
                                  target->space_params.MaxBitmap;
     /* If the device is not a printer class device, it won't support saved-pages */
@@ -6817,10 +6798,7 @@ c_pdf14trans_write(const gs_composite_t	* pct, byte * data, uint * psize,
     int pdf14_needed = cdev->pdf14_needed;
     int trans_group_level = cdev->pdf14_trans_group_level;
     int smask_level = cdev->pdf14_smask_level;
-    bool has_tags = device_encodes_tags((gx_device *)cdev);
-    int bits_per_comp = ((cdev->color_info.depth - has_tags*8) /
-                         cdev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(cdev);
 
     code = dev_proc((gx_device *) cdev, get_profile)((gx_device *) cdev,
                                                      &dev_profile);
@@ -7746,9 +7724,7 @@ get_pdf14_clist_device_proto(gx_device * dev, pdf14_clist_device ** pdevproto,
                 pdf14_determine_default_blend_cs(dev, use_pdf14_accum,
                                                  &using_blend_cs);
     bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     switch (dev_cs) {
         case PDF14_DeviceGray:
@@ -7866,9 +7842,7 @@ pdf14_create_clist_device(gs_memory_t *mem, gs_gstate * pgs,
     gsicc_rendering_param_t render_cond;
     cmm_dev_profile_t *dev_profile;
     uchar k;
-    int bits_per_comp = ((target->color_info.depth - has_tags*8) /
-                         target->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(target);
 
     code = dev_proc(target, get_profile)(target,  &dev_profile);
     if (code < 0)
@@ -8099,10 +8073,7 @@ pdf14_clist_create_compositor(gx_device	* dev, gx_device ** pcdev,
     pdf14_clist_device * pdev = (pdf14_clist_device *)dev;
     int code, is_pdf14_compositor;
     const gs_pdf14trans_t * pdf14pct = (const gs_pdf14trans_t *) pct;
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((pdev->color_info.depth - has_tags*8) /
-                         pdev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     /* We only handle a few PDF 1.4 transparency operations */
     if ((is_pdf14_compositor = gs_is_pdf14trans_compositor(pct)) != 0) {
diff --git a/base/gdevprn.c b/base/gdevprn.c
index cc71e5cc7..b94542805 100644
--- a/base/gdevprn.c
+++ b/base/gdevprn.c
@@ -384,10 +384,7 @@ gdev_prn_allocate(gx_device *pdev, gdev_prn_space_params *new_space_params,
     gs_memory_t *buffer_memory =
         (ppdev->buffer_memory == 0 ? pdev->memory->non_gc_memory :
          ppdev->buffer_memory);
-    bool has_tags = device_encodes_tags(pdev);
-    int bits_per_comp = ((pdev->color_info.depth - has_tags*8) /
-                         pdev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(pdev);
 
     /* If reallocate, find allocated memory & tear down buffer device */
     if (reallocate)
diff --git a/base/gsptype1.c b/base/gsptype1.c
index da96e8b9c..f81a63c10 100644
--- a/base/gsptype1.c
+++ b/base/gsptype1.c
@@ -2047,10 +2047,7 @@ gx_dc_pattern_read(
     int code, l;
     tile_trans_clist_info_t trans_info = { { { 0 } } };
     int cache_space_needed;
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     if (offset == 0) {
         pdevc->mask.id = gx_no_bitmap_id;
diff --git a/base/gstrans.c b/base/gstrans.c
index fd4d299ac..44ed3a7d3 100644
--- a/base/gstrans.c
+++ b/base/gstrans.c
@@ -558,10 +558,7 @@ gs_begin_transparency_mask(gs_gstate * pgs,
     int i, code;
     gs_color_space *blend_color_space;
     gsicc_manager_t *icc_manager = pgs->icc_manager;
-    bool has_tags = device_encodes_tags(pgs->device);
-    int bits_per_comp = ((pgs->device->color_info.depth - has_tags*8) /
-                         pgs->device->color_info.num_components);
-    int deep = bits_per_comp > 8;
+    bool deep = device_is_deep(pgs->device);
 
     if (check_for_nontrans_pattern(pgs,
                   (unsigned char *)"gs_pop_transparency_state")) {
diff --git a/base/gxclthrd.c b/base/gxclthrd.c
index 0cfd30b96..833f9a5e6 100644
--- a/base/gxclthrd.c
+++ b/base/gxclthrd.c
@@ -302,10 +302,7 @@ clist_setup_render_threads(gx_device *dev, int y, gx_process_page_options_t *opt
     /* this will be increased by the measured profile storage and icclinks (estimated).		  */
     int reserve_size = 2 * 1024 * 1024 + (gx_ht_cache_default_bits_size() * dev->color_info.num_components);
     clist_icctable_entry_t *curr_entry;
-    bool has_tags = device_encodes_tags(dev);
-    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
-                         dev->color_info.num_components);
-    bool deep = bits_per_comp > 8;
+    bool deep = device_is_deep(dev);
 
     crdev->num_render_threads = pdev->num_render_threads_requested;
 
diff --git a/base/gxdevcli.h b/base/gxdevcli.h
index 33f7c3724..5524ff81f 100644
--- a/base/gxdevcli.h
+++ b/base/gxdevcli.h
@@ -1835,6 +1835,21 @@ static inline bool device_encodes_tags(const gx_device *dev)
     return (dev->graphics_type_tag & GS_DEVICE_ENCODES_TAGS) != 0;
 }
 
+static inline bool device_is_deep(const gx_device *dev)
+{
+    bool has_tags = device_encodes_tags(dev);
+    int bits_per_comp = ((dev->color_info.depth - has_tags*8) /
+                         dev->color_info.num_components);
+    if (bits_per_comp > 16)
+        return 1;
+    if (bits_per_comp == 16 && dev->color_info.num_components > 1)
+        return 1;
+    if (bits_per_comp == 8)
+        return 0;
+    return (dev->color_info.max_color > 255 ||
+            dev->color_info.max_gray > 255);
+}
+
 /* A null device.  This is used to temporarily disable output. */
 struct gx_device_null_s {
     gx_device_forward_common;