Expand and alter support for NCLR ICC profile and color names

This commit affects the capabilities of the psdcmyk device
and the tiffsep device.  Much of the discussion below
is now in the documentation, but is included here for completeness.

Normally with an NCLR ICC profile you can specify the
colorants using
-sICCOutputColors="Cyan, Magenta, Yellow, Black, Orange, Violet"
for example for a 6CLR ICC profile.

Now we allow additional names beyond those of the ICC profile
component count.  In this case, those names will be installed
into the tiffsep or psdcmyk device list of colors, following
the ICC profile colors.

The number of spot colors (those that go beyond the standard CMYK colors)
allowed by tiffsep or psdcmyk can be set using -dMaxSpots=#.  The default
value for this is currently set to 10 (GS_SOFT_MAX_SPOTS).

As an example consider the case where we wish to use a
6CLR ICC profile that includes Orange and Violet, but
need the device to include a specialty color such as
Varnish, which does not appear in the document and is not
handled by the 6CLR ICC profile.  In addition, we desire
to allow one more spot color of the document to come through
to our device.  For this case using
 -sICCOutputColors="Cyan, Magenta, Yellow, Black, Orange, Violet, Varnish"
   -dMaxSpots= 4
   -sOutputICCProfile=My_6CLR_Profile.icc
would provide the desired outcome.

Note that it is up to the device or through the use of
-sNamedProfile to involve the setting of any values
in the Varnish channel.  However, if an All color value
is encountered in the document, the Varnish component will
have its value set as will the Orange and Violet components
(Likewise if a spot color named Varnish is
encountered in the document the Varnish component will
be used for the values).

Finally this commit adds a debug option --debug=clist-color
which will provide clist writing and reading debug information
for the reading and writing of devn color values to and from
the clist.  This is helpful to debug cases where the pdf14 color
model changes due to group color space changes, which can be
difficult to track down when there is a mismatch in the read and
write of the color values.

Thanks to Ray Johnston for his help on much of this effort.

7 files changed, 148 insertions, 30 deletions

diff --git a/base/gdbflags.h b/base/gdbflags.h
index 073ca73f1..8f044b4ae 100644
--- a/base/gdbflags.h
+++ b/base/gdbflags.h
@@ -32,7 +32,7 @@ FLAG(epo_details, 5, 0, "Erasepage Optimization tracing"),
 FLAG(epo_install_only, 6, 0, "Erasepage Optimization install only (for debugging subclass)"),
 FLAG(init_details, 7, 0, "Language initialisation (detail)"),
 FLAG(overprint, 8, 0, "Overprint"),
-UNUSED(9)
+FLAG(clist_color, 9, 0, "Clist color"),
 UNUSED(10)
 UNUSED(11)
 UNUSED(12)
diff --git a/base/gdevdevn.h b/base/gdevdevn.h
index fb34e1c01..554bcbedc 100644
--- a/base/gdevdevn.h
+++ b/base/gdevdevn.h
@@ -21,9 +21,6 @@
 #include "gxblend.h"
 #include "gsequivc.h"
 
-/* See Comments in gdevtsep.c or gdevpsd.c as to the purpose of this */
-#define LIMIT_TO_ICC 1
-
 /*
  * Type definitions associated with the fixed color model names.
  */
diff --git a/base/gdevp14.c b/base/gdevp14.c
index 0eabc39d5..b212f5dc8 100644
--- a/base/gdevp14.c
+++ b/base/gdevp14.c
@@ -3112,6 +3112,48 @@ pdf14_blend_image_mixed_buffer16(byte* buf_ptr_, int width, int height, int rows
     }
 }
 
+static pdf14_buf*
+insert_empty_planes(pdf14_ctx* ctx, pdf14_buf** src_buf, int num_new_planes, int insert_index)
+{
+    int planestride = (*src_buf)->planestride;
+    int src_n_planes = (*src_buf)->n_planes;
+    int src_n_chan = (*src_buf)->n_chan;
+    int des_n_planes = src_n_planes + num_new_planes;
+    int des_n_chan = src_n_chan + num_new_planes;
+    byte *src_ptr = (*src_buf)->data;
+    byte* des_ptr;
+    byte *des_data;
+    bool deep = ctx->deep;
+
+    des_data = gs_alloc_bytes(ctx->memory,
+        (size_t)planestride * des_n_planes + CAL_SLOP,
+        "insert_empty_planes");
+    if (des_data == NULL)
+        return NULL;
+
+    des_ptr = des_data;
+
+    /* First copy portion prior to insert point */
+    memcpy(des_ptr, src_ptr, (planestride * insert_index) << deep);
+
+    /* New planes */
+    des_ptr += (planestride * insert_index) << deep;
+    src_ptr += (planestride * insert_index) << deep;
+    memset(des_ptr, 0, (planestride * num_new_planes) << deep);
+
+    /* Extra planes (i.e. doc spots, tags) */
+    des_ptr += (planestride * num_new_planes) << deep;
+    memcpy(des_ptr, src_ptr, (planestride * (src_n_planes - insert_index)) << deep);
+
+    /* Set up buffer structure */
+    gs_free_object(ctx->memory, (*src_buf)->data, "insert_empty_planes");
+    (*src_buf)->n_planes = des_n_planes;
+    (*src_buf)->n_chan = des_n_chan;
+    (*src_buf)->data = des_data;
+
+    return *src_buf;
+}
+
 static int
 pdf14_put_blended_image_cmykspot(gx_device* dev, gx_device* target,
     gs_gstate* pgs, pdf14_buf* buf, int planestride_in,
@@ -3355,6 +3397,30 @@ pdf14_put_blended_image_cmykspot(gx_device* dev, gx_device* target,
             tag_offset = buf->has_tags ? buf->n_chan : 0;
         }
 
+        /* We may need to pad the buffers to ensure that any additional spot
+           channels that are not created by the ICC color conversion (or
+           non-conversion if this is not an NCLR profile) get placed properly.
+           It is up to the target device to
+           handle these planes how it sees fit based upon the image data
+           and/or any tags plane during any put image call.  We *could*
+           do something here to possibly communicate through the put_image
+           call where the page related spots start, but that would/could
+           be confusing, especially for long term maintenance. Easier just
+           to have put_image hand all the data */
+        if (dev_target_profile->spotnames != NULL &&
+            dev_target_profile->spotnames->count > des_profile->num_comps) {
+            int num_new_planes = dev_target_profile->spotnames->count - des_profile->num_comps;
+            int insert_index = des_profile->num_comps;
+            pdf14_buf* result;
+
+            result = insert_empty_planes(pdev->ctx, &buf, num_new_planes, insert_index);
+            if (result == NULL)
+                return_error(gs_error_VMerror);
+
+            num_comp = buf->n_chan;
+            tag_offset = buf->has_tags ? buf->n_chan : 0;
+            buf_ptr = buf->data + (rect.p.y - buf->rect.p.y) * buf->rowstride + ((rect.p.x - buf->rect.p.x) << deep);
+        }
 #if RAW_DUMP
         /* Dump after the CS transform */
         dump_raw_buffer_be(target->memory, height, width, buf->n_planes, planestride, rowstride,
@@ -8567,8 +8633,11 @@ gs_pdf14_device_push(gs_memory_t *mem, gs_gstate * pgs,
     p14dev->fillconstantalpha = 1.0;
     p14dev->strokeconstantalpha = 1.0;
 
-    /* Simulated overprint case.  We have to use CMYK-based profile */
-    if (p14dev->overprint_sim && icc_profile->data_cs != gsCMYK) {
+    /* Simulated overprint case.  We have to use CMYK-based profile.  Also if the target
+       profile is NCLR, we are going to use a pdf14 device that is CMYK based and
+       do the mapping to the NCLR profile when the put_image occurs */
+    if ((p14dev->overprint_sim && icc_profile->data_cs != gsCMYK) ||
+        icc_profile->data_cs == gsNCHANNEL) {
         gsicc_adjust_profile_rc(pgs->icc_manager->default_cmyk, 1, "gs_pdf14_device_push");
         gsicc_adjust_profile_rc(p14dev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE],
             -1, "gs_pdf14_device_push");
@@ -9842,7 +9911,11 @@ get_pdf14_clist_device_proto(gx_device          *dev,
             /*
              * The number of components for the PDF14 device is the sum
              * of the process components and the number of spot colors
-             * for the page.
+             * for the page. If we are using an NCLR ICC profile at
+             * the output device, those spot colors are skipped. They
+             * do not appear in the transparency buffer, but appear
+             * during put image transform of the page group to the target
+             * color space.
              */
             if (num_spots >= 0) {
                 pdevproto->devn_params.page_spot_colors = num_spots;
@@ -9971,8 +10044,12 @@ pdf14_create_clist_device(gs_memory_t *mem, gs_gstate * pgs,
         &render_cond);
     if_debug0m('v', mem, "[v]pdf14_create_clist_device\n");
 
-    /* Simulated overprint case.  We have to use CMYK-based profile */
-    if (pdev->overprint_sim && icc_profile->data_cs != gsCMYK) {
+    /* Simulated overprint case.  We have to use CMYK-based profile
+       Also if the target profile is NCLR, we are going to use a pdf14
+       device that is CMYK based and do the mapping to the NCLR profile
+       when the put_image occurs */
+    if ((pdev->overprint_sim && icc_profile->data_cs != gsCMYK) ||
+         icc_profile->data_cs == gsNCHANNEL) {
         gsicc_adjust_profile_rc(pgs->icc_manager->default_cmyk, 1, "pdf14_create_clist_device");
         gsicc_adjust_profile_rc(pdev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE],
             -1, "pdf14_create_clist_device");
@@ -11602,14 +11679,16 @@ c_pdf14trans_clist_read_update(gs_composite_t *	pcte, gx_device	* cdev,
      * device.
      */
     switch (pdf14pct->params.pdf14_op) {
-        case PDF14_PUSH_DEVICE:
-            /* Overprint simulation sets the profile at prototype creation. */
-            if (!p14dev->overprint_sim) {
-                gsicc_adjust_profile_rc(cl_icc_profile, 1, "c_pdf14trans_clist_read_update");
-                gsicc_adjust_profile_rc(p14dev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE],
-                    -1, "c_pdf14trans_clist_read_update");
-                p14dev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE] = cl_icc_profile;
-            }
+    case PDF14_PUSH_DEVICE:
+        /* Overprint simulation sets the profile at prototype creation, as does
+           when the target profile is NCLR. Match the logic in gs_pdf14_device_push */
+        if (!((p14dev->overprint_sim && cl_icc_profile->data_cs != gsCMYK) ||
+            cl_icc_profile->data_cs == gsNCHANNEL)) {
+            gsicc_adjust_profile_rc(cl_icc_profile, 1, "c_pdf14trans_clist_read_update");
+            gsicc_adjust_profile_rc(p14dev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE],
+                -1, "c_pdf14trans_clist_read_update");
+            p14dev->icc_struct->device_profile[GS_DEFAULT_DEVICE_PROFILE] = cl_icc_profile;
+        }
             /*
              * If we are blending using spot colors (i.e. the output device
              * supports spot colors) then we need to transfer
diff --git a/base/gsicc_manage.c b/base/gsicc_manage.c
index 4586441a5..1c3fa91a5 100644
--- a/base/gsicc_manage.c
+++ b/base/gsicc_manage.c
@@ -1586,8 +1586,15 @@ gsicc_set_device_profile_colorants(gx_device *dev, char *name_str)
             char temp_str[DEFAULT_ICC_COLORANT_LENGTH+2];
 
             /* If names are already set then we do not want to set default ones */
-            if (profile_struct->spotnames != NULL)
-                return 0;
+            if (profile_struct->spotnames != NULL) {
+                /* Check if we have at least as many spot names
+                   as there are channels in the proFfile */
+                if (num_comps > profile_struct->spotnames->count) {
+                    gs_warn("ICC profile colorant names count insufficient");
+                    return_error(gs_error_rangecheck);
+                } else
+                    return 0;
+            }
 
             free_str = true;
             /* Assume first 4 are CMYK */
@@ -1933,7 +1940,7 @@ gsicc_set_device_profile(gx_device * pdev, gs_memory_t * mem,
 {
     cmm_profile_t *icc_profile;
     stream *str;
-    int code;
+    int code = 0;
 
     /* This is slightly silly, we have a device method for 'get_profile' we really ought to
      * have one for 'set_profile' as well. In its absence, make sure we are setting the profile
@@ -2047,7 +2054,7 @@ gsicc_set_device_profile(gx_device * pdev, gs_memory_t * mem,
                     break;
                 default:
                     /* NCLR Profile.  Set up default colorant names */
-                    gsicc_set_device_profile_colorants(pdev, NULL);
+                    code = gsicc_set_device_profile_colorants(pdev, NULL);
                     break;
             }
             if_debug1m(gs_debug_flag_icc, mem, "[icc] Profile data CS is %d\n",
@@ -2055,7 +2062,7 @@ gsicc_set_device_profile(gx_device * pdev, gs_memory_t * mem,
         } else
             return gs_rethrow(-1, "cannot find device profile");
     }
-    return 0;
+    return code;
 }
 
 /* Set the icc profile in the gs_color_space object */
diff --git a/base/gstrans.c b/base/gstrans.c
index 782b48463..d810524a0 100644
--- a/base/gstrans.c
+++ b/base/gstrans.c
@@ -810,6 +810,18 @@ gs_push_pdf14trans_device(gs_gstate * pgs, bool is_pattern, bool retain,
     if (depth < 0)
         params.overprint_sim_push = true;
 
+    /* If we have an NCLR ICC profile, the extra spot colorants do not
+       get included in the transparency buffers. This is also true
+       for any extra colorant names listed, which go beyond the profile.
+       Finally, we could have a CMYK profile with colorants listed, that
+       go beyond CMYK. To detect, simply look at dev_profile->spotnames */
+    if (dev_profile->spotnames != NULL && dev_profile->spotnames->count > 4) {
+        /* Making an assumption here, that list is CMYK + extra. */
+        int delta = dev_profile->spotnames->count - 4;
+        params.num_spot_colors_int -= delta;
+        params.num_spot_colors -= delta;
+    }
+
     /* If we happen to be in a situation where we are going out to a device
        whose profile is CIELAB then we will need to make sure that we
        do our blending in RGB and convert to CIELAB when we do the put_image
diff --git a/base/gxcmap.c b/base/gxcmap.c
index a42908c23..26d16c172 100644
--- a/base/gxcmap.c
+++ b/base/gxcmap.c
@@ -1269,19 +1269,26 @@ cmap_rgb_alpha_direct(frac r, frac g, frac b, frac alpha, gx_device_color * pdc,
  *    pcolor_component_map - Map from DeviceN to the Devices colorants.
  *        A negative value indicates component is not to be mapped.
  *    plist - Pointer to list for mapped components
+ *    num_comps - num_comps that we need to zero (may be more than
+ *                is set if we are mapping values for an NCLR ICC profile
+ *                via an alternate tint transform for a sep value) --
+ *                i.e. cmyk+og values and we may have some spots that
+ *                are supported but may have reached the limit and
+ *                using the alt tint values.  Need to make sure to zero all.
  *
  * Returns:
  *    Mapped components in plist.
  */
 static inline void
 map_components_to_colorants(const frac * pcc,
-        const gs_devicen_color_map * pcolor_component_map, frac * plist)
+        const gs_devicen_color_map * pcolor_component_map, frac * plist,
+        int num_colorants)
 {
-    int i = pcolor_component_map->num_colorants - 1;
+    int i;
     int pos;
 
     /* Clear all output colorants first */
-    for (; i >= 0; i--) {
+    for (i = num_colorants - 1; i >= 0; i--) {
         plist[i] = frac_0;
     }
 
@@ -1446,7 +1453,8 @@ cmap_separation_halftoned(frac all, gx_device_color * pdc,
             cm_comps[i] = comp_value;
     } else {
         /* map to the color model */
-        map_components_to_colorants(&all, &(pgs->color_component_map), cm_comps);
+        map_components_to_colorants(&all, &(pgs->color_component_map), cm_comps,
+            pgs->color_component_map.num_colorants);
     }
 
     if (devicen_has_cmyk(dev, des_profile) &&
@@ -1513,7 +1521,8 @@ cmap_separation_direct(frac all, gx_device_color * pdc, const gs_gstate * pgs,
     }
     else {
         /* map to the color model */
-        map_components_to_colorants(&comp_value, &(pgs->color_component_map), cm_comps);
+        map_components_to_colorants(&comp_value, &(pgs->color_component_map), cm_comps,
+            pgs->color_component_map.num_colorants);
     }
 
     /* Check if we have the standard colorants.  If yes, then we will apply
@@ -1618,7 +1627,8 @@ cmap_devicen_halftoned(const frac * pcc,
     gsicc_extract_profile(dev->graphics_type_tag,
                           dev_profile, &des_profile, &render_cond);
     /* map to the color model */
-    map_components_to_colorants(pcc, &(pgs->color_component_map), cm_comps);
+    map_components_to_colorants(pcc, &(pgs->color_component_map), cm_comps,
+        pgs->color_component_map.num_colorants);
     /* See comments in cmap_devicen_direct for details on below operations */
     if (devicen_has_cmyk(dev, des_profile) &&
         des_profile->data_cs == gsCMYK &&
@@ -1667,9 +1677,10 @@ cmap_devicen_direct(const frac * pcc,
     /* map to the color model */
     if (dev_profile->spotnames != NULL && dev_profile->spotnames->equiv_cmyk_set) {
         map_components_to_colorants(pcc, dev_profile->spotnames->color_map,
-                                    cm_comps);
+                                    cm_comps, pgs->color_component_map.num_colorants);
     } else {
-        map_components_to_colorants(pcc, &(pgs->color_component_map), cm_comps);
+        map_components_to_colorants(pcc, &(pgs->color_component_map), cm_comps,
+            pgs->color_component_map.num_colorants);
     }
     /*  Check if we have the standard colorants.  If yes, then we will apply
        ICC color management to those colorants. To understand why, consider
diff --git a/base/gxdcolor.c b/base/gxdcolor.c
index cead03096..4150d5260 100644
--- a/base/gxdcolor.c
+++ b/base/gxdcolor.c
@@ -563,14 +563,20 @@ gx_devn_write_color(
     uchar ncomps = cdev->clist_color_info.num_components; /* Could be different than target if 1.4 device */
     gx_color_index  mask = 0x1, comp_bits = 0;
 
+    if_debug1m(gs_debug_flag_clist_color, dev->memory,
+        "[clist_color] Writing devn color, %d components [ ", ncomps);
+
     /* First find the number of non zero values */
     for (i = 0; i < ncomps; i++, mask <<= 1) {
+        if_debug1m(gs_debug_flag_clist_color, dev->memory,
+            "%d ", pdevc->colors.devn.values[i]);
         if (pdevc->colors.devn.values[i] != 0) {
             comp_bits |= mask;
             count++;
         }
     }
     mask = comp_bits;
+    if_debug0m(gs_debug_flag_clist_color, dev->memory, "]\n");
 
     num_bytes1 = sizeof(gx_color_index);
     num_bytes = num_bytes1 + count * 2 + 1; /* One for the tag byte */
@@ -706,6 +712,9 @@ gx_devn_read_color(
     pos++;
     num_bytes++;
 
+    if_debug1m(gs_debug_flag_clist_color, dev->memory,
+        "[clist_color] Reading devn color, %d components [ ", ncomps);
+
     /* Now the data */
     for (i = 0; i < ncomps; i++) {
         if (mask & 1) {
@@ -717,8 +726,11 @@ gx_devn_read_color(
         } else {
             values[i] = 0;
         }
+        if_debug1m(gs_debug_flag_clist_color, dev->memory,
+            "%d ", values[i]);
         mask >>= 1;
     }
+    if_debug0m(gs_debug_flag_clist_color, dev->memory, "]\n");
     return num_bytes;
 }