data0type.h, row0sel.c:

  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length
ha_innodb.cc:
  Fix a crash in true VARCHARs in test-innodb: we passed a wrong pointer to the column conversion in an UPDATE
rowid_order_innodb.result, ps_3innodb.result, innodb.result, endspace.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs


mysql-test/r/endspace.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/ps_3innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/rowid_order_innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
sql/ha_innodb.cc:
  Fix a crash in true VARCHARs in test-innodb: we passed a wrong pointer to the column conversion in an UPDATE
innobase/row/row0sel.c:
  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length
innobase/include/data0type.h:
  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length

7 files changed, 53 insertions, 33 deletions

diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h
index b5120e22041..edae34cc205 100644
--- a/innobase/include/data0type.h
+++ b/innobase/include/data0type.h
@@ -401,11 +401,20 @@ sym_tab_add_null_lit() */
 
 struct dtype_struct{
 	ulint	mtype;		/* main data type */
-	ulint	prtype;		/* precise type; MySQL data type */
+	ulint	prtype;		/* precise type; MySQL data type, charset code,
+				flags to indicate nullability, signedness,
+				whether this is a binary string, whether this
+				is a true VARCHAR where MySQL uses 2 bytes to
+				store the length */
 
 	/* the remaining fields do not affect alphabetical ordering: */
 
-	ulint	len;		/* length */
+	ulint	len;		/* length; for MySQL data this is
+				field->pack_length(), except that for a
+				>= 5.0.3 type true VARCHAR this is the
+				maximum byte length of the string data
+				(in addition to the string, MySQL uses 1 or
+				2 bytes to store the string length) */
 	ulint	prec;		/* precision */
 
 	ulint	mbminlen;	/* minimum length of a character, in bytes */
diff --git a/innobase/row/row0sel.c b/innobase/row/row0sel.c
index ce8789aa3fc..10fc89bbb37 100644
--- a/innobase/row/row0sel.c
+++ b/innobase/row/row0sel.c
@@ -2018,7 +2018,8 @@ Converts a key value stored in MySQL format to an Innobase dtuple. The last
 field of the key value may be just a prefix of a fixed length field: hence
 the parameter key_len. But currently we do not allow search keys where the
 last field is only a prefix of the full key field len and print a warning if
-such appears. */
+such appears. A counterpart of this function is
+ha_innobase::store_key_val_for_row() in ha_innodb.cc. */
 
 void
 row_sel_convert_mysql_key_to_innobase(
@@ -2099,13 +2100,10 @@ row_sel_convert_mysql_key_to_innobase(
 		type = dfield_get_type(dfield)->mtype;
 
 		/* Calculate data length and data field total length */
-
-		if (type == DATA_BLOB ||
-			dtype_get_mysql_type(dfield_get_type(dfield))
-					== DATA_MYSQL_TRUE_VARCHAR) {
-
-			/* The key field is a column prefix of a BLOB,
-			TEXT, OR TRUE VARCHAR type column */
+		
+		if (type == DATA_BLOB) {
+			/* The key field is a column prefix of a BLOB or
+			TEXT */
 
 			ut_a(field->prefix_len > 0);
 
@@ -2122,11 +2120,9 @@ row_sel_convert_mysql_key_to_innobase(
 				   + 256 * key_ptr[data_offset + 1];
 			data_field_len = data_offset + 2 + field->prefix_len;
 
-			if (type == DATA_BLOB) {
-				data_offset += 2;
-			}
+			data_offset += 2;
 
-			/* now that we know the length, we store the column
+			/* Now that we know the length, we store the column
 			value like it would be a fixed char field */
 
 		} else if (field->prefix_len > 0) {
@@ -2148,6 +2144,18 @@ row_sel_convert_mysql_key_to_innobase(
 			data_field_len = data_offset + data_len;
 		}
 
+ 		if (dtype_get_mysql_type(dfield_get_type(dfield))
+					== DATA_MYSQL_TRUE_VARCHAR) {
+			/* In a MySQL key value format, a true VARCHAR is
+			always preceded by 2 bytes of a length field.
+			dfield_get_type(dfield)->len returns the maximum
+			'payload' len in bytes. That does not include the
+			2 bytes that tell the actual data length. */
+
+			data_len += 2;
+			data_field_len += 2;
+		}
+
 		/* Storing may use at most data_len bytes of buf */
 		
 		if (!is_null) {
diff --git a/mysql-test/r/endspace.result b/mysql-test/r/endspace.result
index b1942409c91..e9396c9a6ed 100644
--- a/mysql-test/r/endspace.result
+++ b/mysql-test/r/endspace.result
@@ -178,7 +178,7 @@ teststring
 teststring
 explain select * from t1 order by text1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	index	NULL	key1	32	NULL	3	Using index
+1	SIMPLE	t1	index	NULL	key1	34	NULL	3	Using index
 alter table t1 modify text1 char(32) binary not null;
 select * from t1 order by text1;
 text1
diff --git a/mysql-test/r/innodb.result b/mysql-test/r/innodb.result
index be73f6986ac..461cf269755 100644
--- a/mysql-test/r/innodb.result
+++ b/mysql-test/r/innodb.result
@@ -1466,13 +1466,13 @@ Error	1146	Table 'test.t4' doesn't exist
 checksum table t1, t2, t3;
 Table	Checksum
 test.t1	2948697075
-test.t2	968604391
-test.t3	968604391
+test.t2	1157260244
+test.t3	1157260244
 checksum table t1, t2, t3 extended;
 Table	Checksum
 test.t1	3092701434
-test.t2	968604391
-test.t3	968604391
+test.t2	1157260244
+test.t3	1157260244
 drop table t1,t2,t3;
 create table t1 (id int,  name char(10) not null,  name2 char(10) not null) engine=innodb;
 insert into t1 values(1,'first','fff'),(2,'second','sss'),(3,'third','ttt');
@@ -1809,6 +1809,5 @@ show variables like "innodb_thread_sleep_delay";
 Variable_name	Value
 innodb_thread_sleep_delay	10000
 create table t1 (v varchar(16384)) engine=innodb;
-ERROR 42000: Column length too big for column 'v' (max = 255); use BLOB instead
 create table t1 (a bit, key(a)) engine=innodb;
 ERROR 42000: The storage engine for the table doesn't support BIT FIELD
diff --git a/mysql-test/r/ps_3innodb.result b/mysql-test/r/ps_3innodb.result
index 940f5e8cbac..9a0927a9a5c 100644
--- a/mysql-test/r/ps_3innodb.result
+++ b/mysql-test/r/ps_3innodb.result
@@ -70,7 +70,7 @@ def	test	t9	t9	c18	c18	1	4	1	Y	32768	0	63
 def	test	t9	t9	c19	c19	1	1	1	Y	32768	0	63
 def	test	t9	t9	c20	c20	254	1	1	Y	0	0	8
 def	test	t9	t9	c21	c21	254	10	10	Y	0	0	8
-def	test	t9	t9	c22	c22	254	30	30	Y	0	0	8
+def	test	t9	t9	c22	c22	253	30	30	Y	0	0	8
 def	test	t9	t9	c23	c23	252	255	8	Y	144	0	63
 def	test	t9	t9	c24	c24	252	255	8	Y	16	0	8
 def	test	t9	t9	c25	c25	252	65535	4	Y	144	0	63
@@ -1691,8 +1691,8 @@ affected rows: 3
 info: Records: 3  Duplicates: 0  Warnings: 0
 select a,b from t2 order by a ;
 a	b
-3	duplicate
-4	duplicate
+3	duplicate 
+4	duplicate 
 103	three
 delete from t2 ;
 prepare stmt1 from ' insert into t2 (b,a) 
@@ -1710,8 +1710,8 @@ affected rows: 3
 info: Records: 3  Duplicates: 0  Warnings: 0
 select a,b from t2 order by a ;
 a	b
-3	duplicate
-4	duplicate
+3	duplicate 
+4	duplicate 
 103	three
 drop table t2;
 drop table if exists t5 ;
diff --git a/mysql-test/r/rowid_order_innodb.result b/mysql-test/r/rowid_order_innodb.result
index f76002e9cb2..d55029f9064 100644
--- a/mysql-test/r/rowid_order_innodb.result
+++ b/mysql-test/r/rowid_order_innodb.result
@@ -178,9 +178,9 @@ insert into t1 values ('','empt',2,2),
 ('dddd','d--d',2,2);
 select * from t1 force index(key1, key2) where key1 < 3 or key2 < 3;
 pk1	pk2	key1	key2
-	empt	2	2
 a	a--a	2	2
 bb	b--b	2	2
 ccc	c--c	2	2
 dddd	d--d	2	2
+	empt	2	2
 drop table t1;
diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc
index 537908f3472..f33931c6645 100644
--- a/sql/ha_innodb.cc
+++ b/sql/ha_innodb.cc
@@ -2920,6 +2920,7 @@ calc_row_difference(
 	ulint		o_len;
 	ulint		n_len;
 	ulint		col_pack_len;
+	byte*		new_mysql_row_col;
 	byte*	        o_ptr;
         byte*	        n_ptr;
         byte*	        buf;
@@ -2948,10 +2949,17 @@ calc_row_difference(
 		o_ptr = (byte*) old_row + get_field_offset(table, field);
 		n_ptr = (byte*) new_row + get_field_offset(table, field);
 		
+		/* Use new_mysql_row_col and col_pack_len save the values */
+
+		new_mysql_row_col = n_ptr;
 		col_pack_len = field->pack_length();
+
 		o_len = col_pack_len;
 		n_len = col_pack_len;
 
+		/* We use o_ptr and n_ptr to dig up the actual data for
+		comparison. */ 
+
 		field_mysql_type = field->type();
 	
 		col_type = get_innobase_type_from_mysql_type(field);
@@ -3017,15 +3025,11 @@ calc_row_difference(
 						&dfield,
 						(byte*)buf,
 						TRUE,
-						n_ptr,
+						new_mysql_row_col,
 						col_pack_len,
 						prebuilt->table->comp);
-				ufield->new_val.data =
-						dfield_get_data_noninline(
-								&dfield);
-				ufield->new_val.len =
-						dfield_get_len_noninline(
-								&dfield);
+				ufield->new_val.data = dfield.data;
+				ufield->new_val.len = dfield.len;
 			} else {
 				ufield->new_val.data = NULL;
 				ufield->new_val.len = UNIV_SQL_NULL;