Bug#10963: LEFT/RIGHT/SUBSTR/.. string functions returns wrong result \

  on large length
  
Problem:  Most (all) of the numeric inputs were being coerced into
int (32 bit) sized variables.  Works OK for sane inputs; any input
larger than 2^32 (or 2^31 for signed vars) exihibited predictable
wrapping behavior (up to about 10^18) and then started having really
strange behaviour past that point (since the conversion to 64 bit int
from the DECIMAL type can do weird things on out of range numbers).

Solution: 1)  Add many tests.  2)  Convert input from (u)long type to
(u)longlong.  3)  Do (sometimes multiple) sanity checks on input,
keeping in mind that sometimes a negative longlong is not a negative
longlong (if the unsigned_flag is set).  4) Emulate existing behavior
w/rt negative and "small" out-of-bounds values.


mysql-test/r/func_str.result:
  Additional test results for #10963
mysql-test/t/func_str.test:
  Additional test results for #10963
sql/item_func.cc:
  Used larger type for counting, to avoid truncation.
sql/item_strfunc.cc:
  Fix for #10963, including comments and cleaned up logic

1 files changed, 189 insertions, 0 deletions

diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test
index 45415882ac7..9622de96143 100644
--- a/mysql-test/t/func_str.test
+++ b/mysql-test/t/func_str.test
@@ -780,4 +780,193 @@ SELECT * FROM t1 INNER JOIN t2 ON code=id
 
 DROP TABLE t1,t2;
 
+#
+# Bug #10963
+# 4294967296 18446744073709551616
+
+select locate('he','hello',-2);
+select locate('lo','hello',-4294967295);
+select locate('lo','hello',4294967295);
+select locate('lo','hello',-4294967296);
+select locate('lo','hello',4294967296);
+select locate('lo','hello',-4294967297);
+select locate('lo','hello',4294967297);
+select locate('lo','hello',-18446744073709551615);
+select locate('lo','hello',18446744073709551615);
+select locate('lo','hello',-18446744073709551616);
+select locate('lo','hello',18446744073709551616);
+select locate('lo','hello',-18446744073709551617);
+select locate('lo','hello',18446744073709551617);
+
+select left('hello', 10);
+select left('hello', 0);
+select left('hello', -1);
+select left('hello', -4294967295);
+select left('hello', 4294967295);
+select left('hello', -4294967296);
+select left('hello', 4294967296);
+select left('hello', -4294967297);
+select left('hello', 4294967297);
+select left('hello', -18446744073709551615);
+select left('hello', 18446744073709551615);
+select left('hello', -18446744073709551616);
+select left('hello', 18446744073709551616);
+select left('hello', -18446744073709551617);
+select left('hello', 18446744073709551617);
+
+select right('hello', 10);
+select right('hello', 0);
+select right('hello', -1);
+select right('hello', -4294967295);
+select right('hello', 4294967295);
+select right('hello', -4294967296);
+select right('hello', 4294967296);
+select right('hello', -4294967297);
+select right('hello', 4294967297);
+select right('hello', -18446744073709551615);
+select right('hello', 18446744073709551615);
+select right('hello', -18446744073709551616);
+select right('hello', 18446744073709551616);
+select right('hello', -18446744073709551617);
+select right('hello', 18446744073709551617);
+
+select substring('hello', 2, -1);
+
+select substring('hello', -1, 1);
+select substring('hello', -2, 1);
+select substring('hello', -4294967295, 1);
+select substring('hello', 4294967295, 1);
+select substring('hello', -4294967296, 1);
+select substring('hello', 4294967296, 1);
+select substring('hello', -4294967297, 1);
+select substring('hello', 4294967297, 1);
+select substring('hello', -18446744073709551615, 1);
+select substring('hello', 18446744073709551615, 1);
+select substring('hello', -18446744073709551616, 1);
+select substring('hello', 18446744073709551616, 1);
+select substring('hello', -18446744073709551617, 1);
+select substring('hello', 18446744073709551617, 1);
+select substring('hello', 1, -1);
+select substring('hello', 1, -4294967295);
+select substring('hello', 1, 4294967295);
+select substring('hello', 1, -4294967296);
+select substring('hello', 1, 4294967296);
+select substring('hello', 1, -4294967297);
+select substring('hello', 1, 4294967297);
+select substring('hello', 1, -18446744073709551615);
+select substring('hello', 1, 18446744073709551615);
+select substring('hello', 1, -18446744073709551616);
+select substring('hello', 1, 18446744073709551616);
+select substring('hello', 1, -18446744073709551617);
+select substring('hello', 1, 18446744073709551617);
+select substring('hello', -1, -1);
+select substring('hello', -4294967295, -4294967295);
+select substring('hello', 4294967295, 4294967295);
+select substring('hello', -4294967296, -4294967296);
+select substring('hello', 4294967296, 4294967296);
+select substring('hello', -4294967297, -4294967297);
+select substring('hello', 4294967297, 4294967297);
+select substring('hello', -18446744073709551615, -18446744073709551615);
+select substring('hello', 18446744073709551615, 18446744073709551615);
+select substring('hello', -18446744073709551616, -18446744073709551616);
+select substring('hello', 18446744073709551616, 18446744073709551616);
+select substring('hello', -18446744073709551617, -18446744073709551617);
+select substring('hello', 18446744073709551617, 18446744073709551617);
+
+select insert('hello', -1, 1, 'hi');
+select insert('hello', -4294967295, 1, 'hi');
+select insert('hello', 4294967295, 1, 'hi');
+select insert('hello', -4294967296, 1, 'hi');
+select insert('hello', 4294967296, 1, 'hi');
+select insert('hello', -4294967297, 1, 'hi');
+select insert('hello', 4294967297, 1, 'hi');
+select insert('hello', -18446744073709551615, 1, 'hi');
+select insert('hello', 18446744073709551615, 1, 'hi');
+select insert('hello', -18446744073709551616, 1, 'hi');
+select insert('hello', 18446744073709551616, 1, 'hi');
+select insert('hello', -18446744073709551617, 1, 'hi');
+select insert('hello', 18446744073709551617, 1, 'hi');
+select insert('hello', 1, -1, 'hi');
+select insert('hello', 1, -4294967295, 'hi');
+select insert('hello', 1, 4294967295, 'hi');
+select insert('hello', 1, -4294967296, 'hi');
+select insert('hello', 1, 4294967296, 'hi');
+select insert('hello', 1, -4294967297, 'hi');
+select insert('hello', 1, 4294967297, 'hi');
+select insert('hello', 1, -18446744073709551615, 'hi');
+select insert('hello', 1, 18446744073709551615, 'hi');
+select insert('hello', 1, -18446744073709551616, 'hi');
+select insert('hello', 1, 18446744073709551616, 'hi');
+select insert('hello', 1, -18446744073709551617, 'hi');
+select insert('hello', 1, 18446744073709551617, 'hi');
+select insert('hello', -1, -1, 'hi');
+select insert('hello', -4294967295, -4294967295, 'hi');
+select insert('hello', 4294967295, 4294967295, 'hi');
+select insert('hello', -4294967296, -4294967296, 'hi');
+select insert('hello', 4294967296, 4294967296, 'hi');
+select insert('hello', -4294967297, -4294967297, 'hi');
+select insert('hello', 4294967297, 4294967297, 'hi');
+select insert('hello', -18446744073709551615, -18446744073709551615, 'hi');
+select insert('hello', 18446744073709551615, 18446744073709551615, 'hi');
+select insert('hello', -18446744073709551616, -18446744073709551616, 'hi');
+select insert('hello', 18446744073709551616, 18446744073709551616, 'hi');
+select insert('hello', -18446744073709551617, -18446744073709551617, 'hi');
+select insert('hello', 18446744073709551617, 18446744073709551617, 'hi');
+
+select repeat('hello', -1);
+select repeat('hello', -4294967295);
+select repeat('hello', 4294967295);
+select repeat('hello', -4294967296);
+select repeat('hello', 4294967296);
+select repeat('hello', -4294967297);
+select repeat('hello', 4294967297);
+select repeat('hello', -18446744073709551615);
+select repeat('hello', 18446744073709551615);
+select repeat('hello', -18446744073709551616);
+select repeat('hello', 18446744073709551616);
+select repeat('hello', -18446744073709551617);
+select repeat('hello', 18446744073709551617);
+
+select space(-1);
+select space(-4294967295);
+select space(4294967295);
+select space(-4294967296);
+select space(4294967296);
+select space(-4294967297);
+select space(4294967297);
+select space(-18446744073709551615);
+select space(18446744073709551615);
+select space(-18446744073709551616);
+select space(18446744073709551616);
+select space(-18446744073709551617);
+select space(18446744073709551617);
+
+select rpad('hello', -1, '1');
+select rpad('hello', -4294967295, '1');
+select rpad('hello', 4294967295, '1');
+select rpad('hello', -4294967296, '1');
+select rpad('hello', 4294967296, '1');
+select rpad('hello', -4294967297, '1');
+select rpad('hello', 4294967297, '1');
+select rpad('hello', -18446744073709551615, '1');
+select rpad('hello', 18446744073709551615, '1');
+select rpad('hello', -18446744073709551616, '1');
+select rpad('hello', 18446744073709551616, '1');
+select rpad('hello', -18446744073709551617, '1');
+select rpad('hello', 18446744073709551617, '1');
+
+select lpad('hello', -1, '1');
+select lpad('hello', -4294967295, '1');
+select lpad('hello', 4294967295, '1');
+select lpad('hello', -4294967296, '1');
+select lpad('hello', 4294967296, '1');
+select lpad('hello', -4294967297, '1');
+select lpad('hello', 4294967297, '1');
+select lpad('hello', -18446744073709551615, '1');
+select lpad('hello', 18446744073709551615, '1');
+select lpad('hello', -18446744073709551616, '1');
+select lpad('hello', 18446744073709551616, '1');
+select lpad('hello', -18446744073709551617, '1');
+select lpad('hello', 18446744073709551617, '1');
+
 --echo End of 5.0 tests