data0type.h, row0sel.c:

  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length
ha_innodb.cc:
  Fix a crash in true VARCHARs in test-innodb: we passed a wrong pointer to the column conversion in an UPDATE
rowid_order_innodb.result, ps_3innodb.result, innodb.result, endspace.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs


mysql-test/r/endspace.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/ps_3innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
mysql-test/r/rowid_order_innodb.result:
  Edit InnoDB test results to reflect the arrival of true VARCHARs
sql/ha_innodb.cc:
  Fix a crash in true VARCHARs in test-innodb: we passed a wrong pointer to the column conversion in an UPDATE
innobase/row/row0sel.c:
  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length
innobase/include/data0type.h:
  Fix a crash in a simple search with a key: the dtype->len of a true VARCHAR is the payload maximum len in bytes: it does not include the 2 bytes MySQL uses to store the string length

2 files changed, 31 insertions, 14 deletions

diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h
index b5120e22041..edae34cc205 100644
--- a/innobase/include/data0type.h
+++ b/innobase/include/data0type.h
@@ -401,11 +401,20 @@ sym_tab_add_null_lit() */
 
 struct dtype_struct{
 	ulint	mtype;		/* main data type */
-	ulint	prtype;		/* precise type; MySQL data type */
+	ulint	prtype;		/* precise type; MySQL data type, charset code,
+				flags to indicate nullability, signedness,
+				whether this is a binary string, whether this
+				is a true VARCHAR where MySQL uses 2 bytes to
+				store the length */
 
 	/* the remaining fields do not affect alphabetical ordering: */
 
-	ulint	len;		/* length */
+	ulint	len;		/* length; for MySQL data this is
+				field->pack_length(), except that for a
+				>= 5.0.3 type true VARCHAR this is the
+				maximum byte length of the string data
+				(in addition to the string, MySQL uses 1 or
+				2 bytes to store the string length) */
 	ulint	prec;		/* precision */
 
 	ulint	mbminlen;	/* minimum length of a character, in bytes */
diff --git a/innobase/row/row0sel.c b/innobase/row/row0sel.c
index ce8789aa3fc..10fc89bbb37 100644
--- a/innobase/row/row0sel.c
+++ b/innobase/row/row0sel.c
@@ -2018,7 +2018,8 @@ Converts a key value stored in MySQL format to an Innobase dtuple. The last
 field of the key value may be just a prefix of a fixed length field: hence
 the parameter key_len. But currently we do not allow search keys where the
 last field is only a prefix of the full key field len and print a warning if
-such appears. */
+such appears. A counterpart of this function is
+ha_innobase::store_key_val_for_row() in ha_innodb.cc. */
 
 void
 row_sel_convert_mysql_key_to_innobase(
@@ -2099,13 +2100,10 @@ row_sel_convert_mysql_key_to_innobase(
 		type = dfield_get_type(dfield)->mtype;
 
 		/* Calculate data length and data field total length */
-
-		if (type == DATA_BLOB ||
-			dtype_get_mysql_type(dfield_get_type(dfield))
-					== DATA_MYSQL_TRUE_VARCHAR) {
-
-			/* The key field is a column prefix of a BLOB,
-			TEXT, OR TRUE VARCHAR type column */
+		
+		if (type == DATA_BLOB) {
+			/* The key field is a column prefix of a BLOB or
+			TEXT */
 
 			ut_a(field->prefix_len > 0);
 
@@ -2122,11 +2120,9 @@ row_sel_convert_mysql_key_to_innobase(
 				   + 256 * key_ptr[data_offset + 1];
 			data_field_len = data_offset + 2 + field->prefix_len;
 
-			if (type == DATA_BLOB) {
-				data_offset += 2;
-			}
+			data_offset += 2;
 
-			/* now that we know the length, we store the column
+			/* Now that we know the length, we store the column
 			value like it would be a fixed char field */
 
 		} else if (field->prefix_len > 0) {
@@ -2148,6 +2144,18 @@ row_sel_convert_mysql_key_to_innobase(
 			data_field_len = data_offset + data_len;
 		}
 
+ 		if (dtype_get_mysql_type(dfield_get_type(dfield))
+					== DATA_MYSQL_TRUE_VARCHAR) {
+			/* In a MySQL key value format, a true VARCHAR is
+			always preceded by 2 bytes of a length field.
+			dfield_get_type(dfield)->len returns the maximum
+			'payload' len in bytes. That does not include the
+			2 bytes that tell the actual data length. */
+
+			data_len += 2;
+			data_field_len += 2;
+		}
+
 		/* Storing may use at most data_len bytes of buf */
 		
 		if (!is_null) {