Convert NS face colors to RGBA when comparing with frame values

The NS port uses indexes into a color table to specify the colors of
faces, whereas frames use RGBA pixel values.  In
extend_face_to_end_of_line the two needed to be compared to ensure
that the backgrounds of certain faces are not extended to the edge of
the window, which was failing because of this difference, thus causing
a visual difference with other platforms.  Convert from index to RGBA
when doing such comparisons.

* src/dispextern.h (FACE_COLOR_TO_PIXEL) [HAVE_NS]: New macro.  Call
ns_color_index_to_rgba under NS only.

* src/nsgui.h: Add prototype for ns_color_index_to_rgba.

* src/nsterm.m (ns_color_index_to_rgba): New function.  Converts a
color_table entry to corresponding RGBA pixel value.

* src/xdisp.c (extend_face_to_end_of_line): Call FACE_COLOR_TO_PIXEL
on face background color when comparing with frame color.

4 files changed, 24 insertions, 3 deletions

diff --git a/src/dispextern.h b/src/dispextern.h
index 579665c2ff8..776d14080e5 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -74,10 +74,13 @@ typedef HDC XImagePtr_or_DC;
 
 #ifdef HAVE_NS
 #include "nsgui.h"
+#define FACE_COLOR_TO_PIXEL(face_color, frame) ns_color_index_to_rgba(face_color, frame)
 /* Following typedef needed to accommodate the MSDOS port, believe it or not.  */
 typedef struct ns_display_info Display_Info;
 typedef Pixmap XImagePtr;
 typedef XImagePtr XImagePtr_or_DC;
+#else
+#define FACE_COLOR_TO_PIXEL(face_color, frame) face_color
 #endif
 
 #ifdef HAVE_WINDOW_SYSTEM
diff --git a/src/nsgui.h b/src/nsgui.h
index 4e7d7d35daa..f858fa7a14a 100644
--- a/src/nsgui.h
+++ b/src/nsgui.h
@@ -73,6 +73,8 @@ typedef unichar XChar2b;
 #define XCHAR2B_BYTE2(chp) \
   (*(chp) & 0x00ff)
 
+/* Used in xdisp.c when comparing faces and frame colors.  */
+extern unsigned long ns_color_index_to_rgba(int idx, struct frame *f);
 
 /* XXX: xfaces requires these structures, but the question is are we
         forced to use them?  */
diff --git a/src/nsterm.m b/src/nsterm.m
index 07978c0d3b8..6ba867d27c0 100644
--- a/src/nsterm.m
+++ b/src/nsterm.m
@@ -2332,6 +2332,22 @@ ns_lisp_to_color (Lisp_Object color, NSColor **col)
   return 1;
 }
 
+/* Convert an index into the color table into an RGBA value.  Used in
+   xdisp.c:extend_face_to_end_of_line when comparing faces and frame
+   color values.  */
+
+unsigned long
+ns_color_index_to_rgba(int idx, struct frame *f)
+{
+  NSColor *col;
+  col = ns_lookup_indexed_color (idx, f);
+
+  EmacsCGFloat r, g, b, a;
+  [col getRed: &r green: &g blue: &b alpha: &a];
+
+  return ARGB_TO_ULONG((int)(a*255),
+                       (int)(r*255), (int)(g*255), (int)(b*255));
+}
 
 void
 ns_query_color(void *col, XColor *color_def, int setPixel)
diff --git a/src/xdisp.c b/src/xdisp.c
index a0113a05190..9a0752f2671 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -20287,7 +20287,7 @@ extend_face_to_end_of_line (struct it *it)
   if (FRAME_WINDOW_P (f)
       && MATRIX_ROW_DISPLAYS_TEXT_P (it->glyph_row)
       && face->box == FACE_NO_BOX
-      && face->background == FRAME_BACKGROUND_PIXEL (f)
+      && FACE_COLOR_TO_PIXEL (face->background, f) == FRAME_BACKGROUND_PIXEL (f)
 #ifdef HAVE_WINDOW_SYSTEM
       && !face->stipple
 #endif
@@ -20432,7 +20432,7 @@ extend_face_to_end_of_line (struct it *it)
 	  && (it->glyph_row->used[LEFT_MARGIN_AREA]
 	      < WINDOW_LEFT_MARGIN_WIDTH (it->w))
 	  && !it->glyph_row->mode_line_p
-	  && default_face->background != FRAME_BACKGROUND_PIXEL (f))
+	  && FACE_COLOR_TO_PIXEL (face->background, f) != FRAME_BACKGROUND_PIXEL (f))
 	{
 	  struct glyph *g = it->glyph_row->glyphs[LEFT_MARGIN_AREA];
 	  struct glyph *e = g + it->glyph_row->used[LEFT_MARGIN_AREA];
@@ -20473,7 +20473,7 @@ extend_face_to_end_of_line (struct it *it)
 	  && (it->glyph_row->used[RIGHT_MARGIN_AREA]
 	      < WINDOW_RIGHT_MARGIN_WIDTH (it->w))
 	  && !it->glyph_row->mode_line_p
-	  && default_face->background != FRAME_BACKGROUND_PIXEL (f))
+	  && FACE_COLOR_TO_PIXEL (face->background, f) != FRAME_BACKGROUND_PIXEL (f))
 	{
 	  struct glyph *g = it->glyph_row->glyphs[RIGHT_MARGIN_AREA];
 	  struct glyph *e = g + it->glyph_row->used[RIGHT_MARGIN_AREA];