Fixes for:

BUG#13519696 - 62940: SELECT RESULTS VARY WITH VERSION AND
WITH/WITHOUT INDEX RANGE SCAN
BUG#13453382 - REGRESSION SINCE 5.1.39, RANGE OPTIMIZER WRONG
RESULTS WITH DECIMAL CONVERSION
BUG#13463488 - 63437: CHAR & BETWEEN WITH INDEX RETURNS WRONG
RESULT AFTER MYSQL 5.1.
Those are all cases where the range optimizer got it wrong
with > and >=.

3 files changed, 195 insertions, 2 deletions

diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result
index ae63edf87b9..5a964baf2c2 100644
--- a/mysql-test/r/range.result
+++ b/mysql-test/r/range.result
@@ -1767,4 +1767,121 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 SELECT * FROM t1, t1 as t2 WHERE t1.i4 BETWEEN t2.pk AND t2.pk;
 pk	i4	pk	i4
 DROP TABLE t1;
+#
+# BUG#13519696 - 62940: SELECT RESULTS VARY WITH VERSION AND
+# WITH/WITHOUT INDEX RANGE SCAN
+#
+create table t1 (id int unsigned not null auto_increment primary key);
+insert into t1 values (null);
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+insert into t1 select null from t1;
+create table t2 (
+id int unsigned not null auto_increment,
+val decimal(5,3) not null,
+primary key (id,val),
+unique key (val,id),
+unique key (id));
+insert into t2 select null,id*0.0009 from t1;
+select count(val) from t2 ignore index (val) where val > 0.1155;
+count(val)
+128
+select count(val) from t2 force index (val)  where val > 0.1155;
+count(val)
+128
+drop table t2, t1;
+#
+# BUG#13453382 - REGRESSION SINCE 5.1.39, RANGE OPTIMIZER WRONG
+# RESULTS WITH DECIMAL CONVERSION
+#
+create table t1 (a int,b int,c int,primary key (a,c));
+insert into t1 values (1,1,2),(1,1,3),(1,1,4);
+select convert(3, signed integer) > 2.9;
+convert(3, signed integer) > 2.9
+1
+select * from t1 force  index (primary) where a=1 and c>= 2.9;
+a	b	c
+1	1	3
+1	1	4
+select * from t1 ignore index (primary) where a=1 and c>= 2.9;
+a	b	c
+1	1	3
+1	1	4
+select * from t1 force  index (primary) where a=1 and c> 2.9;
+a	b	c
+1	1	3
+1	1	4
+select * from t1 ignore index (primary) where a=1 and c> 2.9;
+a	b	c
+1	1	3
+1	1	4
+drop table t1;
+#
+# BUG#13463488 - 63437: CHAR & BETWEEN WITH INDEX RETURNS WRONG
+# RESULT AFTER MYSQL 5.1.
+#
+CREATE TABLE t1(
+F1 CHAR(5) NOT NULL,
+F2 CHAR(5) NOT NULL,
+F3 CHAR(5) NOT NULL,
+PRIMARY KEY(F1),
+INDEX IDX_F2(F2)
+);
+INSERT INTO t1 VALUES
+('A','A','A'),('AA','AA','AA'),('AAA','AAA','AAA'),
+('AAAA','AAAA','AAAA'),('AAAAA','AAAAA','AAAAA');
+SELECT * FROM t1 WHERE F1 = 'A    ';
+F1	F2	F3
+A	A	A
+SELECT * FROM t1 IGNORE INDEX(PRIMARY) WHERE F1 = 'A    ';
+F1	F2	F3
+A	A	A
+SELECT * FROM t1 WHERE F1 >= 'A    ';
+F1	F2	F3
+A	A	A
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+SELECT * FROM t1 WHERE F1 > 'A    ';
+F1	F2	F3
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+SELECT * FROM t1 WHERE F1 BETWEEN 'A    ' AND 'AAAAA';
+F1	F2	F3
+A	A	A
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+SELECT * FROM t1 WHERE F2 BETWEEN 'A    ' AND 'AAAAA';
+F1	F2	F3
+A	A	A
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+SELECT * FROM t1 WHERE F3 BETWEEN 'A    ' AND 'AAAAA';
+F1	F2	F3
+A	A	A
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+SELECT * FROM t1 IGNORE INDEX(PRIMARY) WHERE F1 BETWEEN 'A    ' AND
+'AAAAA';
+F1	F2	F3
+A	A	A
+AA	AA	AA
+AAA	AAA	AAA
+AAAA	AAAA	AAAA
+AAAAA	AAAAA	AAAAA
+DROP TABLE t1;
 End of 5.1 tests
diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test
index 6c9320b708a..30d15798336 100644
--- a/mysql-test/t/range.test
+++ b/mysql-test/t/range.test
@@ -1392,4 +1392,80 @@ SELECT * FROM t1, t1 as t2 WHERE t1.i4 BETWEEN t2.pk AND t2.pk;
 
 DROP TABLE t1;
 
+--echo #
+--echo # BUG#13519696 - 62940: SELECT RESULTS VARY WITH VERSION AND
+--echo # WITH/WITHOUT INDEX RANGE SCAN
+--echo #
+
+create table t1 (id int unsigned not null auto_increment primary key);
+insert into t1 values (null);  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1;  
+insert into t1 select null from t1; 
+create table t2 (
+ id int unsigned not null auto_increment,
+ val decimal(5,3) not null,
+ primary key (id,val),
+ unique key (val,id),
+ unique key (id));  
+--disable_warnings
+insert into t2 select null,id*0.0009 from t1;
+--enable_warnings
+
+select count(val) from t2 ignore index (val) where val > 0.1155;
+select count(val) from t2 force index (val)  where val > 0.1155;
+
+drop table t2, t1;
+
+--echo #
+--echo # BUG#13453382 - REGRESSION SINCE 5.1.39, RANGE OPTIMIZER WRONG
+--echo # RESULTS WITH DECIMAL CONVERSION
+--echo #
+
+create table t1 (a int,b int,c int,primary key (a,c));
+insert into t1 values (1,1,2),(1,1,3),(1,1,4);
+# show that the integer 3 is bigger than the decimal 2.9,
+# which should also apply to comparing "c" with 2.9
+# when c is 3.
+select convert(3, signed integer) > 2.9;
+select * from t1 force  index (primary) where a=1 and c>= 2.9;
+select * from t1 ignore index (primary) where a=1 and c>= 2.9;
+select * from t1 force  index (primary) where a=1 and c> 2.9;
+select * from t1 ignore index (primary) where a=1 and c> 2.9;
+drop table t1;
+
+--echo #
+--echo # BUG#13463488 - 63437: CHAR & BETWEEN WITH INDEX RETURNS WRONG
+--echo # RESULT AFTER MYSQL 5.1.
+--echo #
+
+CREATE TABLE t1(
+ F1 CHAR(5) NOT NULL,
+ F2 CHAR(5) NOT NULL,
+ F3 CHAR(5) NOT NULL,
+ PRIMARY KEY(F1),
+ INDEX IDX_F2(F2)
+);
+
+INSERT INTO t1 VALUES
+('A','A','A'),('AA','AA','AA'),('AAA','AAA','AAA'),
+('AAAA','AAAA','AAAA'),('AAAAA','AAAAA','AAAAA');
+
+SELECT * FROM t1 WHERE F1 = 'A    ';
+SELECT * FROM t1 IGNORE INDEX(PRIMARY) WHERE F1 = 'A    ';
+SELECT * FROM t1 WHERE F1 >= 'A    ';
+SELECT * FROM t1 WHERE F1 > 'A    ';
+SELECT * FROM t1 WHERE F1 BETWEEN 'A    ' AND 'AAAAA';
+SELECT * FROM t1 WHERE F2 BETWEEN 'A    ' AND 'AAAAA';
+SELECT * FROM t1 WHERE F3 BETWEEN 'A    ' AND 'AAAAA';
+SELECT * FROM t1 IGNORE INDEX(PRIMARY) WHERE F1 BETWEEN 'A    ' AND
+'AAAAA';
+
+DROP TABLE t1;
+
 --echo End of 5.1 tests
diff --git a/sql/item.cc b/sql/item.cc
index 930c5d7426e..f4eb418b700 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -7006,7 +7006,7 @@ int stored_field_cmp_to_item(THD *thd, Field *field, Item *item)
 
       return my_time_compare(&field_time, &item_time);
     }
-    return stringcmp(field_result, item_result);
+    return sortcmp(field_result, item_result, field->charset());
   }
   if (res_type == INT_RESULT)
     return 0;					// Both are of type int
@@ -7018,7 +7018,7 @@ int stored_field_cmp_to_item(THD *thd, Field *field, Item *item)
     if (item->null_value)
       return 0;
     field_val= field->val_decimal(&field_buf);
-    return my_decimal_cmp(item_val, field_val);
+    return my_decimal_cmp(field_val, item_val);
   }
   double result= item->val_real();
   if (item->null_value)