Start working on improving UTF-8 support.

This change adds support for 16+bit unicode. This started from a
reading of http://www.cl.cam.ac.uk/~mgk25/ucs/utf-8-history.txt, and
taking help from some code there. There are some issues regarding
refreshing the screen contents, and the copy/search functions need
to be updated, but this is a good first step.

7 files changed, 140 insertions, 62 deletions

diff --git a/src/acconfig.h b/src/acconfig.h
index 2e46985..4431a82 100644
--- a/src/acconfig.h
+++ b/src/acconfig.h
@@ -129,7 +129,7 @@
  * define COLOR to include ansi color support. This may expose
  *        a bug in x11r6-color-xterm.
  * define DW_CHARS to include support for double-width character
- *        sets.
+ *        (i.e. multi-byte) sets.
  * define ENCODINGS to include support for encodings like euc or big5.
  *        Needs FONT to work.
  * define UTF8 if you want support for UTF-8 encoding.
diff --git a/src/ansi.c b/src/ansi.c
index f9e0cc8..32135c8 100644
--- a/src/ansi.c
+++ b/src/ansi.c
@@ -381,7 +381,14 @@ register int len;
 	    {
 	      c = FromUtf8(c, &curr->w_decodestate);
 	      if (c == -1)
-		continue;
+		{
+#if 0
+		  FILE *f = fopen("/tmp/debug/utf-8", "a");
+		  fprintf(f, " %d ", ch);
+		  fclose(f);
+#endif
+		  continue;
+		}
 	      if (c == -2)
 		{
 		  c = UCS_REPL;
@@ -391,6 +398,14 @@ register int len;
 		}
 	      if (c > 0xff)
 		debug1("read UNICODE %04x\n", c);
+#if 0
+	      if (c & 0xffff0000)
+		{
+		  FILE *f = fopen("/tmp/debug/utf-8", "a");
+		  fprintf(f, " %d ", ch);
+		  fclose(f);
+		}
+#endif
 	    }
 #endif
 
@@ -2183,7 +2198,7 @@ static void
 FillWithEs()
 {
   register int i;
-  register unsigned char *p, *ep;
+  register unsigned int *p, *ep;
 
   LClearAll(&curr->w_layer, 1);
   curr->w_y = curr->w_x = 0;
@@ -2237,7 +2252,7 @@ int l;
 static void
 FindAKA()
 {
-  register unsigned char *cp, *line;
+  register unsigned int *cp, *line;
   register struct win *wp = curr;
   register int len = strlen(wp->w_akabuf);
   int y;
@@ -2256,7 +2271,7 @@ FindAKA()
 		goto try_line;
 	      return;
 	    }
-	  if (strncmp((char *)cp, wp->w_akabuf, len) == 0)
+	  if (strncmp((char *)cp, wp->w_akabuf, len) == 0)	/* FIXME: */
 	    break;
 	  cp++;
 	}
@@ -2277,7 +2292,7 @@ FindAKA()
 	    line = cp;
 	  len--;
 	}
-      ChangeAKA(wp, (char *)line, cp - line);
+      ChangeAKA(wp, (char *)line, cp - line);	/* FIXME */
     }
   else
     wp->w_autoaka = 0;
@@ -2352,11 +2367,11 @@ struct mchar *mc;
 	}
     }
 #ifdef FONT
-  if (mc->font && ml->font == null)
+  if (mc->font && (unsigned char *)ml->font == null)
     {
-      if ((ml->font = (unsigned char *)calloc(p->w_width + 1, 1)) == 0)
+      if ((ml->font = (unsigned int *)calloc(p->w_width + 1, sizeof(int))) == 0)
 	{
-	  ml->font = null;
+	  ml->font = (unsigned int *)null;
 	  p->w_FontL = p->w_charsets[p->w_ss ? p->w_ss : p->w_Charset] = 0;
 	  p->w_FontR = p->w_charsets[p->w_ss ? p->w_ss : p->w_CharsetR] = 0;
 	  mc->font = p->w_rend.font  = 0;
@@ -2497,9 +2512,9 @@ int n, ys, ye, bce;
 	    free(ml->attr);
 	  ml->attr = null;
 #ifdef FONT
-	  if (ml->font != null)
+	  if (ml->font != (unsigned int *)null)
 	    free(ml->font);
-	  ml->font = null;
+	  ml->font = (unsigned int *)null;
 #endif
 #ifdef COLOR
 	  if (ml->color != null)
@@ -2542,9 +2557,9 @@ int n, ys, ye, bce;
 	    free(ml->attr);
 	  ml->attr = null;
 #ifdef FONT
-	  if (ml->font != null)
+	  if (ml->font != (unsigned int *)null)
 	    free(ml->font);
-	  ml->font = null;
+	  ml->font = (unsigned int *)null;
 #endif
 #ifdef COLOR
 	  if (ml->color != null)
@@ -2740,6 +2755,7 @@ int x, y;
   struct mline *ml;
   int i;
   unsigned char *b;
+  unsigned int  *ip;
 
   if (n <= 0)
     return;
@@ -2752,9 +2768,9 @@ int x, y;
   for (i = n; i-- > 0;)
     *b++ = r->attr;
 #ifdef FONT
-  b = ml->font + x;
+  ip = ml->font + x;
   for (i = n; i-- > 0;)
-    *b++ = r->font;
+    *ip++ = r->font;
 #endif
 #ifdef COLOR
   b = ml->color + x;
@@ -2805,7 +2821,7 @@ WAddLineToHist(wp, ml)
 struct win *wp;
 struct mline *ml;
 {
-  register unsigned char *q, *o;
+  register void *q, *o;
   struct mline *hml;
 
   if (wp->w_histheight == 0)
@@ -2816,9 +2832,9 @@ struct mline *ml;
   q = ml->attr; o = hml->attr; hml->attr = q; ml->attr = null;
   if (o != null)
     free(o);
- 
+
 #ifdef FONT
-  q = ml->font; o = hml->font; hml->font = q; ml->font = null;
+  q = ml->font; o = hml->font; hml->font = q; ml->font = (unsigned int *)null;
   if (o != null)
     free(o);
 #endif
diff --git a/src/display.c b/src/display.c
index 36368aa..6ec0911 100644
--- a/src/display.c
+++ b/src/display.c
@@ -595,7 +595,7 @@ int c;
 # ifdef UTF8
   if (D_encoding == UTF8)
     {
-      c = (c & 255) | (unsigned char)D_rend.font << 8;
+      c = (c & 255) | (unsigned int )D_rend.font << 8;
 #  ifdef DW_CHARS
       if (D_mbcs)
 	{
diff --git a/src/encoding.c b/src/encoding.c
index 6c1567a..8b99b7c 100644
--- a/src/encoding.c
+++ b/src/encoding.c
@@ -1,4 +1,12 @@
-/* Copyright (c) 1993-2003
+/* Copyright (c) 2010
+ *      Juergen Weigert (jnweiger@immd4.informatik.uni-erlangen.de)
+ *      Sadrul Habib Chowdhury (sadrul@users.sourceforge.net)
+ * Copyright (c) 2008, 2009
+ *      Juergen Weigert (jnweiger@immd4.informatik.uni-erlangen.de)
+ *      Michael Schroeder (mlschroe@immd4.informatik.uni-erlangen.de)
+ *      Micah Cowan (micah@cowan.name)
+ *      Sadrul Habib Chowdhury (sadrul@users.sourceforge.net)
+ * Copyright (c) 1993-2003
  *      Juergen Weigert (jnweiger@immd4.informatik.uni-erlangen.de)
  *      Michael Schroeder (mlschroe@immd4.informatik.uni-erlangen.de)
  * Copyright (c) 1987 Oliver Laumann 
@@ -542,7 +550,7 @@ int from, to;
 
   if (from == to || (from != UTF8 && to != UTF8) || w == 0)
     return ml;
-  if (ml->font == null && encodings[from].deffont == 0)
+  if (ml->font == (unsigned int *)null && encodings[from].deffont == 0)
     return ml;
   if (w > maxlen)
     {
@@ -639,11 +647,54 @@ struct combchar {
 };
 struct combchar **combchars;
 
+/** Thank you Ken! http://www.cl.cam.ac.uk/~mgk25/ucs/utf-8-history.txt */
+typedef struct
+{
+  int	cmask;
+  int	cval;
+  int	shift;
+  long	lmask;
+  long	lval;
+} Tab;
+
+static Tab	tab[] =
+{
+  0x80,	0x00,	0*6,	0x7F,		0,		/* 1 byte sequence */
+  0xE0,	0xC0,	1*6,	0x7FF,		0x80,		/* 2 byte sequence */
+  0xF0,	0xE0,	2*6,	0xFFFF,		0x800,		/* 3 byte sequence */
+  0xF8,	0xF0,	3*6,	0x1FFFFF,	0x10000,	/* 4 byte sequence */
+  0xFC,	0xF8,	4*6,	0x3FFFFFF,	0x200000,	/* 5 byte sequence */
+  0xFE,	0xFC,	5*6,	0x7FFFFFFF,	0x4000000,	/* 6 byte sequence */
+  0,							/* end of table */
+};
+
+
 void
 AddUtf8(c)
 int c;
 {
   ASSERT(D_encoding == UTF8);
+  if (c >= 0xe000)
+    {
+      int l = 0;
+      Tab *t;
+      for (t = tab; t->cmask; t++)
+	{
+	  if (c <= t->lmask)
+	    {
+	      l = t->shift;
+	      AddChar(t->cval | (c>>l));
+	      while (l > 0)
+		{
+		  l -= 6;
+		  AddChar(0x80 | ((c>>l) & 0x3F));
+		}
+	      break;
+	    }
+	}
+      return;
+    }
+
   if (c >= 0xd800 && c < 0xe000 && combchars && combchars[c - 0xd800])
     {
       AddUtf8(combchars[c - 0xd800]->c1);
@@ -758,8 +809,16 @@ int c, *utf8charp;
   *utf8charp = utf8char = (c & 0x80000000) ? c : 0;
   if (utf8char)
     return -1;
+#if 0
+  if (c & 0xffff0000)
+    {
+      FILE *f = fopen("/tmp/debug/utf-8", "a");
+      fprintf(f, " %x ", c);
+      fclose(f);
+    }
   if (c & 0xffff0000)
     c = UCS_REPL;	/* sorry, only know 16bit Unicode */
+#endif
   if (c >= 0xd800 && (c <= 0xdfff || c == 0xfffe || c == 0xffff))
     c = UCS_REPL;	/* illegal code */
   return c;
@@ -803,7 +862,7 @@ int encoding;
 #else
       ml = &p->w_mlines[j];
 #endif
-      if (ml->font == null && encodings[p->w_encoding].deffont == 0)
+      if (ml->font == (unsigned int *)null && encodings[p->w_encoding].deffont == 0)
 	continue;
       for (i = 0; i < p->w_width; i++)
 	{
@@ -812,11 +871,11 @@ int encoding;
 	    c |= encodings[p->w_encoding].deffont << 8;
 	  if (c < 256)
 	    continue;
-	  if (ml->font == null)
+	  if (ml->font == (unsigned int *)null)
 	    {
-	      if ((ml->font = (unsigned char *)calloc(p->w_width + 1, 1)) == 0)
+	      if ((ml->font = (unsigned int *)calloc(p->w_width + 1, sizeof(int))) == 0)
 		{
-		  ml->font = null;
+		  ml->font = (unsigned int *)null;
 		  break;
 		}
 	    }
@@ -1612,7 +1671,8 @@ struct mline *ml;
 int xs, xe;
 int encoding;
 {
-  unsigned char *f, *i;
+  unsigned int *f;
+  unsigned int *i;
   int c, x, dx;
 
   if (encoding == UTF8 || encodings[encoding].deffont == 0)
diff --git a/src/image.h b/src/image.h
index 8622534..f7871b7 100644
--- a/src/image.h
+++ b/src/image.h
@@ -56,18 +56,18 @@
 #endif
 
 struct mchar {
-	 unsigned char image;
+	 unsigned int  image;
 	 unsigned char attr;
-IFFONT(  unsigned char font; )
+IFFONT(  unsigned int  font; )
 IFCOLOR( unsigned char color; )
 IFCOLORX(unsigned char colorx; )
-IFDWCHAR(unsigned char mbcs; )
+IFDWCHAR(unsigned int  mbcs; )
 };
 
 struct mline {
-	 unsigned char *image;
+	 unsigned int  *image;
 	 unsigned char *attr;
-IFFONT(  unsigned char *font; )
+IFFONT(  unsigned int  *font; )
 IFCOLOR( unsigned char *color; )
 IFCOLORX(unsigned char *colorx; )
 };
@@ -75,25 +75,25 @@ IFCOLORX(unsigned char *colorx; )
 
 
 #define save_mline(ml, n) do {						\
-	 bcopy((char *)(ml)->image, (char *)mline_old.image, (n));	\
+	 bcopy((char *)(ml)->image, (char *)mline_old.image, (n) * sizeof(int));	\
 	 bcopy((char *)(ml)->attr,  (char *)mline_old.attr,  (n));	\
-IFFONT(	 bcopy((char *)(ml)->font,  (char *)mline_old.font,  (n));    ) \
+IFFONT(	 bcopy((char *)(ml)->font,  (char *)mline_old.font,  (n) * sizeof(int));    ) \
 IFCOLOR( bcopy((char *)(ml)->color, (char *)mline_old.color, (n));    ) \
 IFCOLORX(bcopy((char *)(ml)->colorx, (char *)mline_old.colorx, (n));  ) \
 } while (0)
 
 #define bcopy_mline(ml, xf, xt, n) do {					       \
-	 bcopy((char *)(ml)->image + (xf), (char *)(ml)->image + (xt), (n));   \
+	 bcopy((char *)((ml)->image + (xf)), (char *)((ml)->image + (xt)), (n) * sizeof(int));   \
 	 bcopy((char *)(ml)->attr  + (xf), (char *)(ml)->attr  + (xt), (n));   \
-IFFONT(	 bcopy((char *)(ml)->font  + (xf), (char *)(ml)->font  + (xt), (n)); ) \
+IFFONT(	 bcopy((char *)((ml)->font  + (xf)), (char *)((ml)->font  + (xt)), (n) * sizeof(int)); ) \
 IFCOLOR( bcopy((char *)(ml)->color + (xf), (char *)(ml)->color + (xt), (n)); ) \
 IFCOLORX(bcopy((char *)(ml)->colorx + (xf), (char *)(ml)->colorx + (xt), (n));) \
 } while (0)
 
 #define clear_mline(ml, x, n) do {					       \
-	 bclear((char *)(ml)->image + (x), (n));			       \
+	 bclear((char *)((ml)->image + (x)), (n) * sizeof(int));			       \
 	 if ((ml)->attr != null) bzero((char *)(ml)->attr  + (x), (n));	       \
-IFFONT(  if ((ml)->font != null) bzero((char *)(ml)->font  + (x), (n));      ) \
+IFFONT(  if ((unsigned char*)(ml)->font != null) bzero((char *)((ml)->font  + (x)), (n) * sizeof(int));      ) \
 IFCOLOR( if ((ml)->color!= null) bzero((char *)(ml)->color + (x), (n));      ) \
 IFCOLORX(if ((ml)->colorx!= null) bzero((char *)(ml)->colorx + (x), (n));    ) \
 } while (0)
diff --git a/src/resize.c b/src/resize.c
index 39f6e28..c463f0f 100644
--- a/src/resize.c
+++ b/src/resize.c
@@ -45,14 +45,15 @@
 static void CheckMaxSize __P((int));
 static void FreeMline  __P((struct mline *));
 static int  AllocMline __P((struct mline *ml, int));
-static void MakeBlankLine __P((unsigned char *, int));
+static void MakeBlankLine __P((unsigned int  *, int));
 static void kaablamm __P((void));
 static int  BcopyMline __P((struct mline *, int, struct mline *, int, int, int));
 static void SwapAltScreen __P((struct win *));
 
 extern struct layer *flayer;
 extern struct display *display, *displays;
-extern unsigned char *blank, *null;
+extern unsigned char *null;
+extern unsigned int  *blank;
 extern struct mline mline_blank, mline_null, mline_old;
 extern struct win *windows;
 extern int Z0width, Z1width;
@@ -63,7 +64,7 @@ struct winsize glwz;
 #endif
 
 static struct mline mline_zero = {
- (unsigned char *)0,
+ (unsigned int  *)0,
  (unsigned char *)0
 #ifdef FONT
  ,(unsigned char *)0
@@ -408,7 +409,7 @@ struct mline *ml;
   if (ml->attr && ml->attr != null)
     free(ml->attr);
 #ifdef FONT
-  if (ml->font && ml->font != null)
+  if (ml->font && ml->font != (unsigned int *)null)
     free(ml->font);
 #endif
 #ifdef COLOR
@@ -427,10 +428,10 @@ AllocMline(ml, w)
 struct mline *ml;
 int w;
 {
-  ml->image = malloc(w);
+  ml->image = malloc(w * sizeof(int));
   ml->attr  = null;
 #ifdef FONT
-  ml->font  = null;
+  ml->font  = (unsigned int *)null;
 #endif
 #ifdef COLOR
   ml->color = null;
@@ -451,7 +452,7 @@ int xf, xt, l, w;
 {
   int r = 0;
 
-  bcopy((char *)mlf->image + xf, (char *)mlt->image + xt, l);
+  bcopy((char *)mlf->image + xf, (char *)mlt->image + xt, l * sizeof(int));
   if (mlf->attr != null && mlt->attr == null)
     {
       if ((mlt->attr = (unsigned char *)calloc(w, 1)) == 0)
@@ -460,13 +461,13 @@ int xf, xt, l, w;
   if (mlt->attr != null)
     bcopy((char *)mlf->attr + xf, (char *)mlt->attr + xt, l);
 #ifdef FONT
-  if (mlf->font != null && mlt->font == null)
+  if (mlf->font != (unsigned int *)null && mlt->font == (unsigned int *)null)
     {
-      if ((mlt->font = (unsigned char *)calloc(w, 1)) == 0)
-	mlt->font = null, r = -1;
+      if ((mlt->font = (unsigned int *)calloc(w, sizeof(int))) == 0)
+	mlt->font = (unsigned int *)null, r = -1;
     }
-  if (mlt->font != null)
-    bcopy((char *)mlf->font + xf, (char *)mlt->font + xt, l);
+  if (mlt->font != (unsigned int *)null)
+    bcopy((char *)mlf->font + xf, (char *)mlt->font + xt, l * sizeof(int));
 #endif
 #ifdef COLOR
   if (mlf->color != null && mlt->color == null)
@@ -497,7 +498,7 @@ CheckMaxSize(wi)
 int wi;
 {
   unsigned char *oldnull = null;
-  unsigned char *oldblank = blank;
+  unsigned int  *oldblank = blank;
   struct win *p;
   int i;
   struct mline *ml;
@@ -507,12 +508,12 @@ int wi;
     return;
   maxwidth = wi;
   debug1("New maxwidth: %d\n", maxwidth);
-  blank = (unsigned char *)xrealloc((char *)blank, maxwidth);
-  null = (unsigned char *)xrealloc((char *)null, maxwidth);
-  mline_old.image = (unsigned char *)xrealloc((char *)mline_old.image, maxwidth);
+  blank = (unsigned int *)xrealloc((char *)blank, maxwidth * sizeof(int));
+  null = (unsigned char *)xrealloc((char *)null, maxwidth * sizeof(int));
+  mline_old.image = (unsigned int *)xrealloc((char *)mline_old.image, maxwidth * sizeof(int));
   mline_old.attr = (unsigned char *)xrealloc((char *)mline_old.attr, maxwidth);
 #ifdef FONT
-  mline_old.font = (unsigned char *)xrealloc((char *)mline_old.font, maxwidth);
+  mline_old.font = (unsigned int  *)xrealloc((char *)mline_old.font, maxwidth * sizeof(int));
 #endif
 #ifdef COLOR
   mline_old.color = (unsigned char *)xrealloc((char *)mline_old.color, maxwidth);
@@ -524,15 +525,15 @@ int wi;
     Panic(0, strnomem);
 
   MakeBlankLine(blank, maxwidth);
-  bzero((char *)null, maxwidth);
+  bzero((char *)null, maxwidth * sizeof(int));
 
   mline_blank.image = blank;
   mline_blank.attr  = null;
-  mline_null.image = null;
+  mline_null.image = (unsigned int *)null;
   mline_null.attr  = null;
 #ifdef FONT
-  mline_blank.font  = null;
-  mline_null.font  = null;
+  mline_blank.font  = (unsigned int *)null;
+  mline_null.font  = (unsigned int *)null;
 #endif
 #ifdef COLOR
   mline_blank.color = null;
@@ -592,7 +593,7 @@ int len;
 
 static void
 MakeBlankLine(p, n)
-register unsigned char *p;
+register unsigned int *p;
 register int n;
 {
   while (n--)
@@ -807,7 +808,7 @@ int wi, he, hi;
 	    {
 	      if (AllocMline(mlt, wi + 1))
 		goto nomem;
-    	      MakeBlankLine(mlt->image + lt, wi - lt);
+	      MakeBlankLine(mlt->image + lt, wi - lt);
 	      mlt->image[wi] = ((oty == ty) ? ' ' : 0);
 	    }
 	  if (BcopyMline(mlf, lf - lx, mlt, lt - lx, lx, wi + 1))
diff --git a/src/window.c b/src/window.c
index 99e0bc7..7fe80e8 100644
--- a/src/window.c
+++ b/src/window.c
@@ -398,9 +398,10 @@ int y, x1, x2, doit;
 struct mchar *rend;
 {
   register int cost, dx;
-  register unsigned char *p, *i;
+  register unsigned char *p;
+  register unsigned int  *i;
 #ifdef FONT
-  register unsigned char *f;
+  register unsigned int  *f;
 #endif
 #ifdef COLOR
   register unsigned char *c;