Bug#20095 Changing length of VARCHAR field with UTF8 collation does not truncate values

Problem: single byte do_varstring1() function was called, which didn't
check limit on "number of character", and checked only "number of bytes".
Fix: adding a multi-byte aware function do_varstring1_mb(),
to limit on "number of characters"


mysql-test/r/ctype_utf8.result:
  Adding test case
mysql-test/t/ctype_utf8.test:
  Adding test case
sql/field_conv.cc:
  Adding missing function to copy VARCHAR strings
  having one length byte.

5 files changed, 181 insertions, 2 deletions

diff --git a/mysql-test/r/bdb_notembedded.result b/mysql-test/r/bdb_notembedded.result
new file mode 100644
index 00000000000..14cb5fad915
--- /dev/null
+++ b/mysql-test/r/bdb_notembedded.result
@@ -0,0 +1,35 @@
+set autocommit=1;
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+drop table bug16206;
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int) engine=         bdb
+f	n	Query	1	n	use `test`; insert into bug16206 values(0)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; BEGIN
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+f	n	Query	1	n	use `test`; COMMIT
+f	n	Query	1	n	use `test`; insert into bug16206 values(3)
+drop table bug16206;
+set autocommit=0;
+End of 5.0 tests
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 1c6bc0e05b6..216b5f393fb 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -1480,6 +1480,49 @@ aa
 xxx
 yyy
 DROP TABLE t1;
+create table t1 (
+a varchar(26) not null
+) default character set utf8;
+insert into t1 (a) values ('abcdefghijklmnopqrstuvwxyz');
+select * from t1;
+a
+abcdefghijklmnopqrstuvwxyz
+alter table t1 change a a varchar(20) character set utf8 not null;
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+select * from t1;
+a
+abcdefghijklmnopqrst
+alter table t1 change a a char(15) character set utf8 not null;
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+select * from t1;
+a
+abcdefghijklmno
+alter table t1 change a a char(10) character set utf8 not null;
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+select * from t1;
+a
+abcdefghij
+alter table t1 change a a varchar(5) character set utf8 not null;
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+select * from t1;
+a
+abcde
+drop table t1;
+create table t1 (
+a varchar(4000) not null
+) default character set utf8;
+insert into t1 values (repeat('a',4000));
+alter table t1 change a a varchar(3000) character set utf8 not null;
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+select length(a) from t1;
+length(a)
+3000
+drop table t1;
 set names utf8;
 select hex(char(1 using utf8));
 hex(char(1 using utf8))
diff --git a/mysql-test/t/bdb_notembedded.test b/mysql-test/t/bdb_notembedded.test
new file mode 100644
index 00000000000..24e64ebbfb2
--- /dev/null
+++ b/mysql-test/t/bdb_notembedded.test
@@ -0,0 +1,38 @@
+-- source include/not_embedded.inc
+-- source include/have_bdb.inc
+
+#
+# Bug #16206: Superfluous COMMIT event in binlog when updating BDB in autocommit mode
+#
+set autocommit=1;
+
+let $VERSION=`select version()`;
+
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+set autocommit=0;
+
+
+--echo End of 5.0 tests
diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test
index 79b73fc7880..c4637d14edc 100644
--- a/mysql-test/t/ctype_utf8.test
+++ b/mysql-test/t/ctype_utf8.test
@@ -1193,6 +1193,41 @@ SELECT DISTINCT id FROM t1 ORDER BY id;
 DROP TABLE t1;
 
 #
+# Bug#20095 Changing length of VARCHAR field with UTF8
+# collation does not truncate values
+#
+create table t1 (
+  a varchar(26) not null
+) default character set utf8;
+insert into t1 (a) values ('abcdefghijklmnopqrstuvwxyz');
+select * from t1;
+# varchar to varchar
+alter table t1 change a a varchar(20) character set utf8 not null;
+select * from t1;
+# varchar to char
+alter table t1 change a a char(15) character set utf8 not null;
+select * from t1;
+# char to char
+alter table t1 change a a char(10) character set utf8 not null;
+select * from t1;
+# char to varchar
+alter table t1 change a a varchar(5) character set utf8 not null;
+select * from t1;
+drop table t1;
+
+#
+# Check that do_varstring2_mb produces a warning
+#
+create table t1 (
+  a varchar(4000) not null
+) default character set utf8;
+insert into t1 values (repeat('a',4000));
+alter table t1 change a a varchar(3000) character set utf8 not null;
+select length(a) from t1;
+drop table t1;
+
+
+#
 #  Bug#10504: Character set does not support traditional mode
 #  Bug#14146: CHAR(...USING ...) and CONVERT(CHAR(...) USING...)
 #             produce different results
diff --git a/sql/field_conv.cc b/sql/field_conv.cc
index 429d914db97..5670b1e0676 100644
--- a/sql/field_conv.cc
+++ b/sql/field_conv.cc
@@ -434,6 +434,26 @@ static void do_varstring1(Copy_field *copy)
 }
 
 
+static void do_varstring1_mb(Copy_field *copy)
+{
+  int well_formed_error;
+  CHARSET_INFO *cs= copy->from_field->charset();
+  uint from_length= (uint) *(uchar*) copy->from_ptr;
+  const char *from_ptr= copy->from_ptr + 1;
+  uint to_char_length= (copy->to_length - 1) / cs->mbmaxlen;
+  uint length= cs->cset->well_formed_len(cs, from_ptr, from_ptr + from_length,
+                                         to_char_length, &well_formed_error);
+  if (length < from_length)
+  {
+    if (current_thd->count_cuted_fields)
+      copy->to_field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN,
+                                  WARN_DATA_TRUNCATED, 1);
+  }
+  *(uchar*) copy->to_ptr= (uchar) length;
+  memcpy(copy->to_ptr + 1, from_ptr, length);
+}
+
+
 static void do_varstring2(Copy_field *copy)
 {
   uint length=uint2korr(copy->from_ptr);
@@ -459,6 +479,12 @@ static void do_varstring2_mb(Copy_field *copy)
   const char *from_beg= copy->from_ptr + HA_KEY_BLOB_LENGTH;
   uint length= cs->cset->well_formed_len(cs, from_beg, from_beg + from_length,
                                          char_length, &well_formed_error);
+  if (length < from_length)
+  {
+    if (current_thd->count_cuted_fields)
+      copy->to_field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN,
+                                  WARN_DATA_TRUNCATED, 1);
+  }  
   int2store(copy->to_ptr, length);
   memcpy(copy->to_ptr+HA_KEY_BLOB_LENGTH, from_beg, length);
 }
@@ -634,8 +660,10 @@ void (*Copy_field::get_copy_func(Field *to,Field *from))(Copy_field*)
           return do_field_string;
         if (to_length != from_length)
           return (((Field_varstring*) to)->length_bytes == 1 ?
-                  do_varstring1 : (from->charset()->mbmaxlen == 1 ?
-                                   do_varstring2 : do_varstring2_mb));
+                  (from->charset()->mbmaxlen == 1 ? do_varstring1 :
+                                                    do_varstring1_mb) :
+                  (from->charset()->mbmaxlen == 1 ? do_varstring2 :
+                                                    do_varstring2_mb));
       }
       else if (to_length < from_length)
 	return (from->charset()->mbmaxlen == 1 ?