Bug#13463415 63502: INCORRECT RESULTS OF BIGINT AND DECIMAL COMPARISON

Bug#11758543 50756: BIGINT '100' MATCHES 1.001E2

Expressions of the form
      BIGINT_COL <compare> <non-integer constant>

      should be done either as decimal, or float.

      Currently however, such comparisons are done as int,
      which means that the constant may be truncated,
      and yield false positives/negatives for all queries
      where compare is '>' '<' '>=' '<=' '=' '!='.

      BIGINT_COL IN <list of contstants>
      and
      BIGINT_COL BETWEEN <constant> AND <constant>
      are also affected.



mysql-test/r/bigint.result:
  New tests.
mysql-test/r/func_in.result:
  BIGINT <=> string comparison should be done as float,
  so a warning for the value 'abc' is appropriate.
mysql-test/t/bigint.test:
  New tests.
sql/item_cmpfunc.cc:
  In convert_constant_item() we verify that the constant item
  can be stored in the given field.
  For BIGINT columns (MYSQL_TYPE_LONGLONG) we must verify that the
  stored constant value is actually comparable as int,
  i.e. that the value was not truncated.
  
  For between: compare as int only if both arguments convert correctly to int.

3 files changed, 113 insertions, 0 deletions

diff --git a/mysql-test/r/bigint.result b/mysql-test/r/bigint.result
index 47a45efa5fd..652c5b96147 100644
--- a/mysql-test/r/bigint.result
+++ b/mysql-test/r/bigint.result
@@ -438,3 +438,67 @@ Level	Code	Message
 Note	1003	select 1 AS `1` from `test`.`t1` where ((`test`.`t1`.`a` = 0) and ('0' = `test`.`t1`.`b`))
 DROP TABLE t1;
 # End of 5.1 tests
+#
+# Bug#13463415 63502: INCORRECT RESULTS OF BIGINT AND DECIMAL COMPARISON
+#
+CREATE TABLE t_bigint(id BIGINT);
+INSERT INTO t_bigint VALUES (1), (2);
+SELECT id, id >= 1.1 FROM t_bigint;
+id	id >= 1.1
+1	0
+2	1
+SELECT id, 1.1 <= id FROM t_bigint;
+id	1.1 <= id
+1	0
+2	1
+SELECT id, id = 1.1 FROM t_bigint;
+id	id = 1.1
+1	0
+2	0
+SELECT id, 1.1 = id FROM t_bigint;
+id	1.1 = id
+1	0
+2	0
+SELECT * from t_bigint WHERE id = 1.1;
+id
+SELECT * from t_bigint WHERE id = 1.1e0;
+id
+SELECT * from t_bigint WHERE id = '1.1';
+id
+SELECT * from t_bigint WHERE id = '1.1e0';
+id
+SELECT * from t_bigint WHERE id IN (1.1, 2.2);
+id
+SELECT * from t_bigint WHERE id IN (1.1e0, 2.2e0);
+id
+SELECT * from t_bigint WHERE id IN ('1.1', '2.2');
+id
+SELECT * from t_bigint WHERE id IN ('1.1e0', '2.2e0');
+id
+SELECT * from t_bigint WHERE id BETWEEN 1.1 AND 1.9;
+id
+SELECT * from t_bigint WHERE id BETWEEN 1.1e0 AND 1.9e0;
+id
+SELECT * from t_bigint WHERE id BETWEEN '1.1' AND '1.9';
+id
+SELECT * from t_bigint WHERE id BETWEEN '1.1e0' AND '1.9e0';
+id
+DROP TABLE t_bigint;
+#
+# Bug#11758543 50756: BIGINT '100' MATCHES 1.001E2
+#
+CREATE TABLE t1 (a BIGINT);
+INSERT INTO t1 VALUES (1);
+SELECT * FROM t1 WHERE coalesce(a) BETWEEN 0 and 0.9;
+a
+SELECT * FROM t1 WHERE coalesce(a)=0.9;
+a
+SELECT * FROM t1 WHERE coalesce(a) in (0.8,0.9);
+a
+SELECT * FROM t1 WHERE a BETWEEN 0 AND 0.9;
+a
+SELECT * FROM t1 WHERE a=0.9;
+a
+SELECT * FROM t1 WHERE a IN (0.8,0.9);
+a
+DROP TABLE t1;
diff --git a/mysql-test/r/func_in.result b/mysql-test/r/func_in.result
index 0b6117581f3..279a18e344e 100644
--- a/mysql-test/r/func_in.result
+++ b/mysql-test/r/func_in.result
@@ -469,6 +469,8 @@ SELECT HEX(a) FROM t2 WHERE a IN
 HEX(a)
 7FFFFFFFFFFFFFFE
 7FFFFFFFFFFFFFFF
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: 'abc'
 CREATE TABLE t3 (a BIGINT UNSIGNED);
 INSERT INTO t3 VALUES (9223372036854775551);
 SELECT HEX(a) FROM t3 WHERE a IN (9223372036854775807, 42);
diff --git a/mysql-test/t/bigint.test b/mysql-test/t/bigint.test
index e19bba971f9..41f33b8a7f2 100644
--- a/mysql-test/t/bigint.test
+++ b/mysql-test/t/bigint.test
@@ -361,4 +361,51 @@ DROP TABLE t1;
 
 --echo # End of 5.1 tests
 
+--echo #
+--echo # Bug#13463415 63502: INCORRECT RESULTS OF BIGINT AND DECIMAL COMPARISON
+--echo #
+
+CREATE TABLE t_bigint(id BIGINT);
+INSERT INTO t_bigint VALUES (1), (2);
+
+SELECT id, id >= 1.1 FROM t_bigint;
+SELECT id, 1.1 <= id FROM t_bigint;
+
+SELECT id, id = 1.1 FROM t_bigint;
+SELECT id, 1.1 = id FROM t_bigint;
+
+SELECT * from t_bigint WHERE id = 1.1;
+SELECT * from t_bigint WHERE id = 1.1e0;
+SELECT * from t_bigint WHERE id = '1.1';
+SELECT * from t_bigint WHERE id = '1.1e0';
+
+SELECT * from t_bigint WHERE id IN (1.1, 2.2);
+SELECT * from t_bigint WHERE id IN (1.1e0, 2.2e0);
+SELECT * from t_bigint WHERE id IN ('1.1', '2.2');
+SELECT * from t_bigint WHERE id IN ('1.1e0', '2.2e0');
+
+SELECT * from t_bigint WHERE id BETWEEN 1.1 AND 1.9;
+SELECT * from t_bigint WHERE id BETWEEN 1.1e0 AND 1.9e0;
+SELECT * from t_bigint WHERE id BETWEEN '1.1' AND '1.9';
+SELECT * from t_bigint WHERE id BETWEEN '1.1e0' AND '1.9e0';
 
+DROP TABLE t_bigint;
+
+--echo #
+--echo # Bug#11758543 50756: BIGINT '100' MATCHES 1.001E2
+--echo #
+
+CREATE TABLE t1 (a BIGINT);
+INSERT INTO t1 VALUES (1);
+
+# a. These queries correctly return 0 rows:
+SELECT * FROM t1 WHERE coalesce(a) BETWEEN 0 and 0.9;
+SELECT * FROM t1 WHERE coalesce(a)=0.9;
+SELECT * FROM t1 WHERE coalesce(a) in (0.8,0.9);
+
+# b. These queries mistakenely returned 1 row:
+SELECT * FROM t1 WHERE a BETWEEN 0 AND 0.9;
+SELECT * FROM t1 WHERE a=0.9;
+SELECT * FROM t1 WHERE a IN (0.8,0.9);
+
+DROP TABLE t1;