Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.

        We cannot cut a line from a polygon. So if the polygon fits the condition, and
        the intersection of a line and the polygon doesn't, we just skip the line.
        That rule wasn't applied if the line start was inside the polygon, which leaded
        to the assertion.

per-file comments:
  mysql-test/r/gis-precise.result
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        test result updated.
  mysql-test/t/gis-precise.test
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        test case added.
  sql/gcalc_tools.cc
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        Don't mark the line as a border if it's inside a polygon that fits the result condition.

3 files changed, 26 insertions, 3 deletions

diff --git a/mysql-test/r/gis-precise.result b/mysql-test/r/gis-precise.result
index 98d081f48e3..d943a534137 100644
--- a/mysql-test/r/gis-precise.result
+++ b/mysql-test/r/gis-precise.result
@@ -279,3 +279,12 @@ AsText(ST_UNION(MultiPolygonFromText('
                      ((2 2, 2 8, 8 8, 8 2, 2 2), (4 4, 4 6, 6 6, 6 4, 4 4)))'),
 MultiPolygonFr
 POLYGON((0 0,0 5,0.555555555555556 5,1 9,2 8,8 8,8 2,7 2,5.33333333333333 2,3 1.125,3 0,0 0),(1 1,1 1.5,1.33333333333333 2,1 2,2 2,2 1.14285714285714,1.75 1,1 1),(3 1.71428571428571,3 2,3.5 2,3 1.71428571428571),(4 4,4 6,4.5 6,5.5 4,4 4))
+SELECT AsText(ST_SYMDIFFERENCE(
+MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))));
+AsText(ST_SYMDIFFERENCE(
+MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))))
+GEOMETRYCOLLECTION(LINESTRING(9 9,8 4,9 0,9 9,7 7),POLYGON((0 0,0 9,7 9,7 0,0 0)),LINESTRING(7 5.28571428571429,8 5,7 8))
diff --git a/mysql-test/t/gis-precise.test b/mysql-test/t/gis-precise.test
index 30445219d90..bf36e6e9628 100644
--- a/mysql-test/t/gis-precise.test
+++ b/mysql-test/t/gis-precise.test
@@ -158,7 +158,7 @@ SELECT Round(ST_AREA(ST_BUFFER( ST_UNION(
          POLYGONFROMTEXT('POLYGON((7 7, 4 7, 2 9, 7 6, 7 7))')), 1)), 6);
 
 
-#buf #804259 Second assertion in Gis_geometry_collection::init_from_opresult
+#bug #804259 Second assertion in Gis_geometry_collection::init_from_opresult
 
 SELECT AsText(ST_UNION(MultiPolygonFromText('
         MULTIPOLYGON(((2 2, 2 8, 8 8, 8 2, 2 2), (4 4, 4 6, 6 6, 6 4, 4 4)),
@@ -168,3 +168,12 @@ MultiPolygonFromText(' MULTIPOLYGON(((0 0, 1 9, 4 6, 0 0)),
                      ((0 5, 3 5, 3 4, 1 4, 1 3, 3 3, 3 0, 0 0, 0 5), (1 1, 2 1, 2 2, 1 2, 1 1)),
                      ((7 7, 4 7, 6 3, 7 2, 7 7)),
                      ((0 5, 3 5, 3 4, 1 4, 1 3, 3 3, 3 0, 0 0, 0 5), (1 1, 2 1, 2 2, 1 2, 1 1))) ')));
+
+
+#bug 801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple
+
+SELECT AsText(ST_SYMDIFFERENCE(
+        MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+        Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))));
+
diff --git a/sql/gcalc_tools.cc b/sql/gcalc_tools.cc
index 2a9090a04a1..d89385fa78c 100644
--- a/sql/gcalc_tools.cc
+++ b/sql/gcalc_tools.cc
@@ -879,7 +879,7 @@ int Gcalc_operation_reducer::count_slice(Gcalc_scan_iterator *si)
     if (!new_t)
       return 1;
     m_fn->invert_state(pi.get_shape());
-    new_t->result_range= prev_state ^ m_fn->count();
+    new_t->result_range= ~prev_state & m_fn->count();
     new_t->next= *at_hook;
     *at_hook= new_t;
     if (new_t->result_range &&
@@ -891,12 +891,17 @@ int Gcalc_operation_reducer::count_slice(Gcalc_scan_iterator *si)
   {
     active_thread *new_t0, *new_t1;
     int fn_result;
+    const Gcalc_heap::Info *p= pi.get_pi();
+    bool line= m_fn->get_shape_kind(p->shape) == Gcalc_function::shape_line;
     if (!(new_t0= new_active_thread()) || !(new_t1= new_active_thread()))
       return 1;
     
     m_fn->invert_state(pi.get_shape());
     fn_result= m_fn->count();
-    new_t0->result_range= new_t1->result_range= prev_state ^ fn_result;
+    if (line)
+      new_t0->result_range= new_t1->result_range= ~prev_state & fn_result;
+    else
+      new_t0->result_range= new_t1->result_range= prev_state ^ fn_result;
     new_t1->next= *at_hook;
     new_t0->next= new_t1;
     *at_hook= new_t0;