Set up the bidi iterator for iterating display strings and overlay strings.

Not tested yet, just compiled.

 src/xdisp.c (handle_single_display_spec, next_overlay_string)
 (get_overlay_strings_1, push_display_prop): Set up the bidi
 iterator for displaying display or overlay strings.
 (forward_to_next_line_start): Don't use the shortcut if
 bidi-iterating.
 (back_to_previous_visible_line_start): If handle_display_prop
 pushed the iterator stack, restore the internal state of the bidi
 iterator by calling bidi_pop_it same number of times.
 (reseat_at_next_visible_line_start): If ON_NEWLINE_P is non-zero,
 and we are bidi-iterating, don't decrement the iterator position;
 instead, set the first_elt flag in the bidi iterator, to produce
 the same effect.
 (reseat_1): Remove redundant setting of string_from_display_prop_p.
 (push_display_prop): xassert that we are iterating a buffer.
 (push_it, pop_it): Save and restore the state of the
 bidi iterator.  Save and restore the bidi_p flag.
 (pop_it): Iterate out of display property for string iteration as
 well.
 (iterate_out_of_display_property): Support iteration over strings.
 (handle_single_display_spec): Set up it->bidi_it for iteration
 over a display string, and call bidi_init_it.
 src/bidi.c (bidi_cache_start_stack, bidi_push_it): Use IT_STACK_SIZE.
 src/dispextern.h (struct iterator_stack_entry): New member bidi_p.
 (struct it): Member bidi_p is now a bit field 1 bit wide.

4 files changed, 214 insertions, 38 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 40576554636..083aaaaa4bd 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,35 @@
+2011-06-25  Eli Zaretskii  <eliz@gnu.org>
+
+	* xdisp.c (handle_single_display_spec, next_overlay_string)
+	(get_overlay_strings_1, push_display_prop): Set up the bidi
+	iterator for displaying display or overlay strings.
+	(forward_to_next_line_start): Don't use the shortcut if
+	bidi-iterating.
+	(back_to_previous_visible_line_start): If handle_display_prop
+	pushed the iterator stack, restore the internal state of the bidi
+	iterator by calling bidi_pop_it same number of times.
+	(reseat_at_next_visible_line_start): If ON_NEWLINE_P is non-zero,
+	and we are bidi-iterating, don't decrement the iterator position;
+	instead, set the first_elt flag in the bidi iterator, to produce
+	the same effect.
+	(reseat_1): Remove redundant setting of string_from_display_prop_p.
+	(push_display_prop): xassert that we are iterating a buffer.
+
+	* bidi.c (bidi_cache_start_stack, bidi_push_it): Use IT_STACK_SIZE.
+
+2011-06-24  Eli Zaretskii  <eliz@gnu.org>
+
+	* xdisp.c (push_it, pop_it): Save and restore the state of the
+	bidi iterator.  Save and restore the bidi_p flag.
+	(pop_it): Iterate out of display property for string iteration as
+	well.
+	(iterate_out_of_display_property): Support iteration over strings.
+	(handle_single_display_spec): Set up it->bidi_it for iteration
+	over a display string, and call bidi_init_it.
+
+	* dispextern.h (struct iterator_stack_entry): New member bidi_p.
+	(struct it): Member bidi_p is now a bit field 1 bit wide.
+
 2011-06-23  Eli Zaretskii  <eliz@gnu.org>
 
 	* dispextern.h (bidi_push_it, bidi_pop_it): Add prototypes.
diff --git a/src/bidi.c b/src/bidi.c
index b03e93df817..b518dd45782 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -548,11 +548,10 @@ bidi_peek_at_next_level (struct bidi_it *bidi_it)
 /***********************************************************************
 	     Pushing and popping the bidi iterator state
  ***********************************************************************/
-/* 10-slot stack for saving the start of the previous level of the
-   cache.  xdisp.c maintains a 5-slot cache for its iterator state,
-   and we need just a little bit more.  */
-#define CACHE_STACK_SIZE 10
-static int bidi_cache_start_stack[CACHE_STACK_SIZE];
+/* 5-slot stack for saving the start of the previous level of the
+   cache.  xdisp.c maintains a 5-slot stack for its iterator state,
+   and we need the same size of our stack.  */
+static int bidi_cache_start_stack[IT_STACK_SIZE];
 static int bidi_cache_sp;
 
 /* Push the bidi iterator state in preparation for reordering a
@@ -569,7 +568,7 @@ bidi_push_it (struct bidi_it *bidi_it)
   memcpy (&bidi_cache[bidi_cache_idx++], bidi_it, sizeof (struct bidi_it));
 
   /* Push the current cache start onto the stack.  */
-  if (bidi_cache_sp >= CACHE_STACK_SIZE)
+  if (bidi_cache_sp >= IT_STACK_SIZE)
     abort ();
   bidi_cache_start_stack[bidi_cache_sp++] = bidi_cache_start;
 
diff --git a/src/dispextern.h b/src/dispextern.h
index acd7862e5dc..d5479c7a64e 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -2157,7 +2157,9 @@ struct it
   Lisp_Object *dpvec, *dpend;
 
   /* Length in bytes of the char that filled dpvec.  A value of zero
-     means that no such character is involved.  */
+     means that no such character is involved.  A negative value means
+     the rest of the line from the current iterator position onwards
+     is hidden by selective display or ellipsis.  */
   int dpvec_char_len;
 
   /* Face id to use for all characters in display vector.  -1 if unused. */
@@ -2253,6 +2255,7 @@ struct it
     unsigned string_from_display_prop_p : 1;
     unsigned display_ellipsis_p : 1;
     unsigned avoid_cursor_p : 1;
+    unsigned bidi_p:1;
     enum line_wrap_method line_wrap;
 
     /* properties from display property that are reset by another display property. */
@@ -2473,7 +2476,7 @@ struct it
 
   /* Non-zero means we need to reorder bidirectional text for display
      in the visual order.  */
-  int bidi_p;
+  unsigned bidi_p : 1;
 
   /* For iterating over bidirectional text.  */
   struct bidi_it bidi_it;
diff --git a/src/xdisp.c b/src/xdisp.c
index 74afa80ada7..dc9a62fc0b6 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -4482,12 +4482,43 @@ handle_single_display_spec (struct it *it, Lisp_Object spec, Lisp_Object object,
 	  it->end_charpos = it->string_nchars = SCHARS (it->string);
 	  it->method = GET_FROM_STRING;
 	  it->stop_charpos = 0;
+	  it->prev_stop = 0;
+	  it->base_level_stop = 0;
 	  it->string_from_display_prop_p = 1;
 	  /* Say that we haven't consumed the characters with
 	     `display' property yet.  The call to pop_it in
 	     set_iterator_to_next will clean this up.  */
 	  if (BUFFERP (object))
 	    *position = start_pos;
+
+	  /* Force paragraph direction to be that of the parent
+	     object.  */
+	  it->paragraph_embedding =
+	    (it->bidi_p ? it->bidi_it.paragraph_dir : L2R);
+
+	  /* Do we need to reorder this display string?  */
+	  if (it->multibyte_p)
+	    {
+	      if (BUFFERP (object))
+		it->bidi_p =
+		  !NILP (BVAR (XBUFFER (object), bidi_display_reordering));
+	      else
+		it->bidi_p =
+		  !NILP (BVAR (&buffer_defaults, bidi_display_reordering));
+	    }
+	  else
+	    it->bidi_p = 0;
+
+	  /* Set up the bidi iterator for this display string.  */
+	  if (it->bidi_p)
+	    {
+	      it->bidi_it.string.lstring = it->string;
+	      it->bidi_it.string.s = NULL;
+	      it->bidi_it.string.schars = it->end_charpos;
+	      it->bidi_it.string.bufpos = bufpos;
+	      it->bidi_it.string.from_disp_str = 1;
+	      bidi_init_it (0, 0, FRAME_WINDOW_P (it->f), &it->bidi_it);
+	    }
 	}
       else if (CONSP (value) && EQ (XCAR (value), Qspace))
 	{
@@ -4839,6 +4870,24 @@ next_overlay_string (struct it *it)
       it->stop_charpos = 0;
       if (it->cmp_it.stop_pos >= 0)
 	it->cmp_it.stop_pos = 0;
+      it->prev_stop = 0;
+      it->base_level_stop = 0;
+
+      /* Do we need to reorder this overlay string?  */
+      it->bidi_p =
+	it->multibyte_p
+	&& !NILP (BVAR (current_buffer, bidi_display_reordering));
+
+      /* Set up the bidi iterator for this overlay string.  */
+      if (it->bidi_p)
+	{
+	  it->bidi_it.string.lstring = it->string;
+	  it->bidi_it.string.s = NULL;
+	  it->bidi_it.string.schars = SCHARS (it->string);
+	  it->bidi_it.string.bufpos = it->overlay_strings_charpos;
+	  it->bidi_it.string.from_disp_str = it->string_from_display_prop_p;
+	  bidi_init_it (0, 0, FRAME_WINDOW_P (it->f), &it->bidi_it);
+	}
     }
 
   CHECK_IT (it);
@@ -5105,8 +5154,32 @@ get_overlay_strings_1 (struct it *it, EMACS_INT charpos, int compute_stop_p)
       it->stop_charpos = 0;
       xassert (STRINGP (it->string));
       it->end_charpos = SCHARS (it->string);
+      it->prev_stop = 0;
+      it->base_level_stop = 0;
       it->multibyte_p = STRING_MULTIBYTE (it->string);
       it->method = GET_FROM_STRING;
+
+      /* Do we need to reorder this overlay string?  */
+      it->bidi_p =
+	it->multibyte_p
+	&& !NILP (BVAR (current_buffer, bidi_display_reordering));
+
+      /* Force paragraph direction to be that of the parent
+	 buffer.  */
+      it->paragraph_embedding = (it->bidi_p ? it->bidi_it.paragraph_dir : L2R);
+
+      /* Set up the bidi iterator for this overlay string.  */
+      if (it->bidi_p)
+	{
+	  EMACS_INT pos = (charpos > 0 ? charpos : IT_CHARPOS (*it));
+
+	  it->bidi_it.string.lstring = it->string;
+	  it->bidi_it.string.s = NULL;
+	  it->bidi_it.string.schars = SCHARS (it->string);
+	  it->bidi_it.string.bufpos = pos;
+	  it->bidi_it.string.from_disp_str = it->string_from_display_prop_p;
+	  bidi_init_it (0, 0, FRAME_WINDOW_P (it->f), &it->bidi_it);
+	}
       return 1;
     }
 
@@ -5181,19 +5254,28 @@ push_it (struct it *it, struct text_pos *position)
   p->string_from_display_prop_p = it->string_from_display_prop_p;
   p->display_ellipsis_p = 0;
   p->line_wrap = it->line_wrap;
+  p->bidi_p = it->bidi_p;
   ++it->sp;
+
+  /* Save the state of the bidi iterator as well. */
+  if (it->bidi_p)
+    bidi_push_it (&it->bidi_it);
 }
 
 static void
 iterate_out_of_display_property (struct it *it)
 {
+  int buffer_p = BUFFERP (it->object);
+  EMACS_INT eob = (buffer_p ? ZV : it->end_charpos);
+  EMACS_INT bob = (buffer_p ? BEGV : 0);
+
   /* Maybe initialize paragraph direction.  If we are at the beginning
      of a new paragraph, next_element_from_buffer may not have a
      chance to do that.  */
-  if (it->bidi_it.first_elt && it->bidi_it.charpos < ZV)
+  if (it->bidi_it.first_elt && it->bidi_it.charpos < eob)
     bidi_paragraph_init (it->paragraph_embedding, &it->bidi_it, 1);
   /* prev_stop can be zero, so check against BEGV as well.  */
-  while (it->bidi_it.charpos >= BEGV
+  while (it->bidi_it.charpos >= bob
 	 && it->prev_stop <= it->bidi_it.charpos
 	 && it->bidi_it.charpos < CHARPOS (it->position))
     bidi_move_to_visually_next (&it->bidi_it);
@@ -5207,7 +5289,10 @@ iterate_out_of_display_property (struct it *it)
     {
       SET_TEXT_POS (it->position,
 		    it->bidi_it.charpos, it->bidi_it.bytepos);
-      it->current.pos = it->position;
+      if (buffer_p)
+	it->current.pos = it->position;
+      else
+	it->current.string_pos = it->position;
     }
 }
 
@@ -5249,18 +5334,6 @@ pop_it (struct it *it)
       break;
     case GET_FROM_BUFFER:
       it->object = it->w->buffer;
-      if (it->bidi_p)
-	{
-	  /* Bidi-iterate until we get out of the portion of text, if
-	     any, covered by a `display' text property or an overlay
-	     with `display' property.  (We cannot just jump there,
-	     because the internal coherency of the bidi iterator state
-	     can not be preserved across such jumps.)  We also must
-	     determine the paragraph base direction if the overlay we
-	     just processed is at the beginning of a new
-	     paragraph.  */
-	  iterate_out_of_display_property (it);
-	}
       break;
     case GET_FROM_STRING:
       it->object = it->string;
@@ -5286,6 +5359,20 @@ pop_it (struct it *it)
   it->voffset = p->voffset;
   it->string_from_display_prop_p = p->string_from_display_prop_p;
   it->line_wrap = p->line_wrap;
+  it->bidi_p = p->bidi_p;
+  if (it->bidi_p)
+    {
+      bidi_pop_it (&it->bidi_it);
+      /* Bidi-iterate until we get out of the portion of text, if any,
+	 covered by a `display' text property or by an overlay with
+	 `display' property.  (We cannot just jump there, because the
+	 internal coherency of the bidi iterator state can not be
+	 preserved across such jumps.)  We also must determine the
+	 paragraph base direction if the overlay we just processed is
+	 at the beginning of a new paragraph.  */
+      if (it->method == GET_FROM_BUFFER || it->method == GET_FROM_STRING)
+	iterate_out_of_display_property (it);
+    }
 }
 
 
@@ -5368,15 +5455,16 @@ forward_to_next_line_start (struct it *it, int *skipped_p)
 
       xassert (!STRINGP (it->string));
 
-      /* If there isn't any `display' property in sight, and no
-	 overlays, we can just use the position of the newline in
-	 buffer text.  */
-      if (it->stop_charpos >= limit
-	  || ((pos = Fnext_single_property_change (make_number (start),
-						   Qdisplay,
-						   Qnil, make_number (limit)),
-	       NILP (pos))
-	      && next_overlay_change (start) == ZV))
+      /* If we are not bidi-reordering, and there isn't any `display'
+	 property in sight, and no overlays, we can just use the
+	 position of the newline in buffer text.  */
+      if (!it->bidi_p
+	  && (it->stop_charpos >= limit
+	      || ((pos = Fnext_single_property_change (make_number (start),
+						       Qdisplay, Qnil,
+						       make_number (limit)),
+		   NILP (pos))
+		  && next_overlay_change (start) == ZV)))
 	{
 	  IT_CHARPOS (*it) = limit;
 	  IT_BYTEPOS (*it) = CHAR_TO_BYTE (limit);
@@ -5456,7 +5544,20 @@ back_to_previous_visible_line_start (struct it *it)
 	    && (OVERLAYP (overlay)
 		? (beg = OVERLAY_POSITION (OVERLAY_START (overlay)))
 		: get_property_and_range (pos, Qdisplay, &val, &beg, &end, Qnil)))
-	  goto replaced;
+	  {
+	    /* If the call to handle_display_prop above pushed the
+	       iterator state, that causes side effects for the bidi
+	       iterator by calling bidi_push_it.  Undo those side
+	       effects.  */
+	    while (it2.sp > 0)
+	      {
+		/* push_it calls bidi_push_it only if the bidi_p flag
+		   is set in the iterator being pushed.  */
+		if (it2.stack[--it2.sp].bidi_p)
+		  bidi_pop_it (&it2.bidi_it);
+	      }
+	    goto replaced;
+	  }
 
 	/* Newline is not replaced by anything -- so we are done.  */
 	break;
@@ -5525,14 +5626,29 @@ reseat_at_next_visible_line_start (struct it *it, int on_newline_p)
 	{
 	  if (IT_STRING_CHARPOS (*it) > 0)
 	    {
-	      --IT_STRING_CHARPOS (*it);
-	      --IT_STRING_BYTEPOS (*it);
+	      if (!it->bidi_p)
+		{
+		  --IT_STRING_CHARPOS (*it);
+		  --IT_STRING_BYTEPOS (*it);
+		}
+	      else
+		/* Setting this flag will cause
+		   bidi_move_to_visually_next not to advance, but
+		   instead deliver the current character (newline),
+		   which is what the ON_NEWLINE_P flag wants.  */
+		it->bidi_it.first_elt = 1;
 	    }
 	}
       else if (IT_CHARPOS (*it) > BEGV)
 	{
-	  --IT_CHARPOS (*it);
-	  --IT_BYTEPOS (*it);
+	  if (!it->bidi_p)
+	    {
+	      --IT_CHARPOS (*it);
+	      --IT_BYTEPOS (*it);
+	    }
+	  /* With bidi iteration, the call to `reseat' will cause
+	     bidi_move_to_visually_next deliver the current character,
+	     the newline, instead of advancing.  */
 	  reseat (it, it->current.pos, 0);
 	}
     }
@@ -5614,7 +5730,6 @@ reseat_1 (struct it *it, struct text_pos pos, int set_stop_p)
   IT_STRING_CHARPOS (*it) = -1;
   IT_STRING_BYTEPOS (*it) = -1;
   it->string = Qnil;
-  it->string_from_display_prop_p = 0;
   it->method = GET_FROM_BUFFER;
   it->object = it->w->buffer;
   it->area = TEXT_AREA;
@@ -17644,6 +17759,8 @@ cursor_row_p (struct glyph_row *row)
 static int
 push_display_prop (struct it *it, Lisp_Object prop)
 {
+  xassert (it->method == GET_FROM_BUFFER);
+
   push_it (it, NULL);
 
   if (STRINGP (prop))
@@ -17661,6 +17778,30 @@ push_display_prop (struct it *it, Lisp_Object prop)
       it->end_charpos = it->string_nchars = SCHARS (it->string);
       it->method = GET_FROM_STRING;
       it->stop_charpos = 0;
+      it->prev_stop = 0;
+      it->base_level_stop = 0;
+      it->string_from_display_prop_p = 1;
+
+      /* Force paragraph direction to be that of the parent
+	 buffer.  */
+      it->paragraph_embedding = (it->bidi_p ? it->bidi_it.paragraph_dir : L2R);
+
+      /* Do we need to reorder this string?  */
+      if (it->multibyte_p)
+	it->bidi_p = !NILP (BVAR (current_buffer, bidi_display_reordering));
+      else
+	it->bidi_p = 0;
+
+      /* Set up the bidi iterator for this display string.  */
+      if (it->bidi_p)
+	{
+	  it->bidi_it.string.lstring = it->string;
+	  it->bidi_it.string.s = NULL;
+	  it->bidi_it.string.schars = it->end_charpos;
+	  it->bidi_it.string.bufpos = IT_CHARPOS (*it);
+	  it->bidi_it.string.from_disp_str = 1;
+	  bidi_init_it (0, 0, FRAME_WINDOW_P (it->f), &it->bidi_it);
+	}
     }
   else if (CONSP (prop) && EQ (XCAR (prop), Qspace))
     {
@@ -17707,6 +17848,7 @@ static void
 handle_line_prefix (struct it *it)
 {
   Lisp_Object prefix;
+
   if (it->continuation_lines_width > 0)
     {
       prefix = get_it_property (it, Qwrap_prefix);