Many files:

  InnoDB true VARCHAR


sql/ha_innodb.h:
  InnoDB true VARCHAR
sql/ha_innodb.cc:
  InnoDB true VARCHAR
innobase/include/data0type.h:
  InnoDB true VARCHAR
innobase/include/que0que.h:
  InnoDB true VARCHAR
innobase/include/row0mysql.h:
  InnoDB true VARCHAR
innobase/include/data0type.ic:
  InnoDB true VARCHAR
innobase/include/row0mysql.ic:
  InnoDB true VARCHAR
innobase/row/row0ins.c:
  InnoDB true VARCHAR
innobase/row/row0mysql.c:
  InnoDB true VARCHAR
innobase/row/row0sel.c:
  InnoDB true VARCHAR
innobase/trx/trx0trx.c:
  InnoDB true VARCHAR

3 files changed, 306 insertions, 81 deletions

diff --git a/innobase/row/row0ins.c b/innobase/row/row0ins.c
index fdbbe993ff0..303fe5749bc 100644
--- a/innobase/row/row0ins.c
+++ b/innobase/row/row0ins.c
@@ -521,6 +521,10 @@ row_ins_cascade_calc_update_vec(
 
 				fixed_size = dtype_get_fixed_size(type);
 
+				/* TODO: pad in UCS-2 with 0x0020.
+				TODO: How does the special truncation of
+				UTF-8 CHAR cols affect this? */
+
 				if (fixed_size
 				    && ufield->new_val.len != UNIV_SQL_NULL
 				    && ufield->new_val.len < fixed_size) {
diff --git a/innobase/row/row0mysql.c b/innobase/row/row0mysql.c
index 2b2b2d83002..b13ba056d85 100644
--- a/innobase/row/row0mysql.c
+++ b/innobase/row/row0mysql.c
@@ -106,20 +106,6 @@ row_mysql_delay_if_needed(void)
 }
 
 /***********************************************************************
-Reads a MySQL format variable-length field (like VARCHAR) length and
-returns pointer to the field data. */
-
-byte*
-row_mysql_read_var_ref_noninline(
-/*=============================*/
-			/* out: field + 2 */
-	ulint*	len,	/* out: variable-length field length */
-	byte*	field)	/* in: field */
-{
-	return(row_mysql_read_var_ref(len, field));
-}
-
-/***********************************************************************
 Frees the blob heap in prebuilt when no longer needed. */
 
 void
@@ -133,6 +119,61 @@ row_mysql_prebuilt_free_blob_heap(
 }
 
 /***********************************************************************
+Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row
+format. */
+
+byte*
+row_mysql_store_true_var_len(
+/*=========================*/
+			/* out: pointer to the data, we skip the 1 or 2 bytes
+			at the start that are used to store the len */
+	byte*	dest,	/* in: where to store */
+	ulint	len,	/* in: length, must fit in two bytes */
+	ulint	lenlen)	/* in: storage length of len: either 1 or 2 bytes */
+{
+	if (lenlen == 2) {
+		ut_a(len < 256 * 256);
+
+		mach_write_to_2_little_endian(dest, len);
+
+		return(dest + 2);
+	}
+
+	ut_a(lenlen == 1);
+	ut_a(len < 256);
+
+	mach_write_to_1(dest, len);
+
+	return(dest + 1);
+}
+
+/***********************************************************************
+Reads a >= 5.0.3 format true VARCHAR length, in the MySQL row format, and
+returns a pointer to the data. */
+
+byte*
+row_mysql_read_true_varchar(
+/*========================*/
+			/* out: pointer to the data, we skip the 1 or 2 bytes
+			at the start that are used to store the len */
+	ulint*	len,	/* out: variable-length field length */
+	byte*	field,	/* in: field in the MySQL format */
+	ulint	lenlen)	/* in: storage length of len: either 1 or 2 bytes */
+{
+	if (lenlen == 2) {
+		*len = mach_read_from_2_little_endian(field);
+
+		return(field + 2);
+	}
+
+	ut_a(lenlen == 1);
+
+	*len = mach_read_from_1(field);
+
+	return(field + 1);
+}
+
+/***********************************************************************
 Stores a reference to a BLOB in the MySQL format. */
 
 void
@@ -191,15 +232,177 @@ row_mysql_read_blob_ref(
 }
 
 /******************************************************************
-Convert a row in the MySQL format to a row in the Innobase format. */
+Stores a non-SQL-NULL field given in the MySQL format in the InnoDB format.
+The counterpart of this function is row_sel_field_store_in_mysql_format() in
+row0sel.c. */
+
+byte*
+row_mysql_store_col_in_innobase_format(
+/*===================================*/
+					/* out: up to which byte we used
+					buf in the conversion */
+	dfield_t*	dfield,		/* in/out: dfield where dtype
+					information must be already set when
+					this function is called! */
+	byte*		buf,		/* in/out: buffer for a converted
+					integer value; this must be at least
+					col_len long then! */
+	ibool		row_format_col,	/* TRUE if the mysql_data is from
+					a MySQL row, FALSE if from a MySQL
+					key value;
+					in MySQL, a true VARCHAR storage
+					format differs in a row and in a
+					key value: in a key value the length
+					is always stored in 2 bytes! */
+	byte*		mysql_data,	/* in: MySQL column value, not
+					SQL NULL; NOTE that dfield may also
+					get a pointer to mysql_data,
+					therefore do not discard this as long
+					as dfield is used! */
+	ulint		col_len,	/* in: MySQL column length; NOTE that
+					this is the storage length of the
+					column in the MySQL format row, not
+					necessarily the length of the actual
+					payload data; if the column is a true
+					VARCHAR then this is irrelevant */
+	ibool		comp)		/* in: TRUE = compact format */
+{
+	byte*		ptr 	= mysql_data;
+	dtype_t*	dtype;
+	ulint		type;
+	ulint		lenlen;
+
+	dtype = dfield_get_type(dfield);
+
+	type = dtype->mtype;
+
+	if (type == DATA_INT) {
+		/* Store integer data in Innobase in a big-endian format,
+		sign bit negated if the data is a signed integer. In MySQL,
+		integers are stored in a little-endian format. */
+
+		ptr = buf + col_len;
+
+		for (;;) {
+			ptr--;
+			*ptr = *mysql_data;
+			if (ptr == buf) {
+				break;
+			}
+			mysql_data++;
+		}
+
+		if (!(dtype->prtype & DATA_UNSIGNED)) {
+
+			*ptr = (byte) (*ptr ^ 128);
+		}
+
+		buf += col_len;
+	} else if ((type == DATA_VARCHAR
+		    || type == DATA_VARMYSQL
+		    || type == DATA_BINARY)) {
+
+		if (dtype_get_mysql_type(dtype) == DATA_MYSQL_TRUE_VARCHAR) {
+			/* The length of the actual data is stored to 1 or 2
+			bytes at the start of the field */
+			
+			if (row_format_col) {
+				if (dtype->prtype & DATA_LONG_TRUE_VARCHAR) {
+					lenlen = 2;
+				} else {
+					lenlen = 1;
+				}
+			} else {
+				/* In a MySQL key value, lenlen is always 2 */
+				lenlen = 2;
+			}
+
+			ptr = row_mysql_read_true_varchar(&col_len, mysql_data,
+								      lenlen);
+		} else {
+			/* Remove trailing spaces from old style VARCHAR
+			columns. */
+
+			/* Handle UCS2 strings differently. */
+			ulint	mbminlen	= dtype_get_mbminlen(dtype);
+
+			ptr = mysql_data;
+
+			if (mbminlen == 2) {
+				/* space=0x0020 */
+				/* Trim "half-chars", just in case. */
+				col_len &= ~1;
+
+				while (col_len >= 2 && ptr[col_len - 2] == 0x00
+						&& ptr[col_len - 1] == 0x20) {
+					col_len -= 2;
+				}
+			} else {
+				ut_a(mbminlen == 1);
+				/* space=0x20 */
+				while (col_len > 0
+						&& ptr[col_len - 1] == 0x20) {
+					col_len--;
+				}
+			}
+		}
+	} else if (comp && type == DATA_MYSQL
+			&& dtype_get_mbminlen(dtype) == 1
+			&& dtype_get_mbmaxlen(dtype) > 1) {
+		/* In some cases we strip trailing spaces from UTF-8 and other
+		multibyte charsets, from FIXED-length CHAR columns, to save
+		space. UTF-8 would otherwise normally use 3 * the string length
+		bytes to store a latin1 string! */
+
+		/* We assume that this CHAR field is encoded in a
+		variable-length character set where spaces have
+		1:1 correspondence to 0x20 bytes, such as UTF-8.
+
+		Consider a CHAR(n) field, a field of n characters.
+		It will contain between n * mbminlen and n * mbmaxlen bytes.
+		We will try to truncate it to n bytes by stripping
+		space padding.  If the field contains single-byte
+		characters only, it will be truncated to n characters.
+		Consider a CHAR(5) field containing the string ".a   "
+		where "." denotes a 3-byte character represented by
+		the bytes "$%&".  After our stripping, the string will
+		be stored as "$%&a " (5 bytes).  The string ".abc "
+		will be stored as "$%&abc" (6 bytes).
+
+		The space padding will be restored in row0sel.c, function
+		row_sel_field_store_in_mysql_format(). */
+
+		ulint		n_chars;
+
+		ut_a(!(dtype_get_len(dtype) % dtype_get_mbmaxlen(dtype)));
+
+		n_chars = dtype_get_len(dtype) / dtype_get_mbmaxlen(dtype);
+
+		/* Strip space padding. */
+		while (col_len > n_chars && ptr[col_len - 1] == 0x20) {
+			col_len--;
+		}
+	} else if (type == DATA_BLOB && row_format_col) {
+
+		ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len);
+	}
+
+	dfield_set_data(dfield, ptr, col_len);
+
+	return(buf);
+}
+
+/******************************************************************
+Convert a row in the MySQL format to a row in the Innobase format. Note that
+the function to convert a MySQL format key value to an InnoDB dtuple is
+row_sel_convert_mysql_key_to_innobase() in row0sel.c. */
 static
 void
 row_mysql_convert_row_to_innobase(
 /*==============================*/
 	dtuple_t*	row,		/* in/out: Innobase row where the
 					field type information is already
-					copied there, or will be copied
-					later */
+					copied there! */
 	row_prebuilt_t*	prebuilt,	/* in: prebuilt struct where template
 					must be of type ROW_MYSQL_WHOLE_ROW */
 	byte*		mysql_rec)	/* in: row in the MySQL format;
@@ -236,10 +439,10 @@ row_mysql_convert_row_to_innobase(
 		row_mysql_store_col_in_innobase_format(dfield,
 					prebuilt->ins_upd_rec_buff
 						+ templ->mysql_col_offset,
+					TRUE, /* MySQL row format data */
 					mysql_rec + templ->mysql_col_offset,
 					templ->mysql_col_len,
-					templ->type, prebuilt->table->comp,
-					templ->is_unsigned);
+					prebuilt->table->comp);
 next_column:
 		;
 	} 
@@ -594,7 +797,8 @@ static
 dtuple_t*
 row_get_prebuilt_insert_row(
 /*========================*/
-					/* out: prebuilt dtuple */
+					/* out: prebuilt dtuple; the column
+					type information is also set in it */ 
 	row_prebuilt_t*	prebuilt)	/* in: prebuilt struct in MySQL
 					handle */
 {
@@ -784,6 +988,7 @@ row_unlock_tables_for_mysql(
 	lock_release_tables_off_kernel(trx);
 	mutex_exit(&kernel_mutex);
 }
+
 /*************************************************************************
 Sets a table lock on the table mentioned in prebuilt. */
 
@@ -962,10 +1167,13 @@ run_again:
 
 	if (err != DB_SUCCESS) {
 		que_thr_stop_for_mysql(thr);
-    thr->lock_state= QUE_THR_LOCK_ROW;
+
+/* TODO: what is this? */ thr->lock_state= QUE_THR_LOCK_ROW;
+
 		was_lock_wait = row_mysql_handle_errors(&err, trx, thr,
 								&savept);
-    thr->lock_state= QUE_THR_LOCK_NOLOCK;
+		thr->lock_state= QUE_THR_LOCK_NOLOCK;
+
 		if (was_lock_wait) {
 			goto run_again;
 		}
diff --git a/innobase/row/row0sel.c b/innobase/row/row0sel.c
index 54dfbe997ce..a09e09342e0 100644
--- a/innobase/row/row0sel.c
+++ b/innobase/row/row0sel.c
@@ -2119,10 +2119,10 @@ row_sel_convert_mysql_key_to_innobase(
 				   + 256 * key_ptr[data_offset + 1];
 			data_field_len = data_offset + 2 + field->prefix_len;
 			data_offset += 2;
-			
-			type = DATA_CHAR; /* now that we know the length, we
-					  store the column value like it would
-					  be a fixed char field */
+
+			/* now that we know the length, we store the column
+			value like it would be a fixed char field */
+
 		} else if (field->prefix_len > 0) {
 			/* Looks like MySQL pads unused end bytes in the
 			prefix with space. Therefore, also in UTF-8, it is ok
@@ -2146,11 +2146,12 @@ row_sel_convert_mysql_key_to_innobase(
 		
 		if (!is_null) {
 		        row_mysql_store_col_in_innobase_format(
-					dfield, buf, key_ptr + data_offset,
-					data_len, type,
-					index->table->comp,
-					dfield_get_type(dfield)->prtype
-							& DATA_UNSIGNED);
+					dfield,
+					buf,
+					FALSE, /* MySQL key value format col */
+					key_ptr + data_offset,
+					data_len,
+					index->table->comp);
 			buf += data_len;
 		}
 
@@ -2225,7 +2226,7 @@ row_sel_store_row_id_to_prebuilt(
 		dict_index_name_print(stderr, prebuilt->trx, index);
 		fprintf(stderr, "\n"
 "InnoDB: Field number %lu, record:\n",
-			(ulong) dict_index_get_sys_col_pos(index, DATA_ROW_ID));
+		    (ulong) dict_index_get_sys_col_pos(index, DATA_ROW_ID));
 		rec_print_new(stderr, index_rec, offsets);
 		putc('\n', stderr);
 		ut_error;
@@ -2235,8 +2236,9 @@ row_sel_store_row_id_to_prebuilt(
 }
 
 /******************************************************************
-Stores a non-SQL-NULL field in the MySQL format. */
-UNIV_INLINE
+Stores a non-SQL-NULL field in the MySQL format. The counterpart of this
+function is row_mysql_store_col_in_innobase_format() in row0mysql.c. */
+static
 void
 row_sel_field_store_in_mysql_format(
 /*================================*/
@@ -2251,6 +2253,8 @@ row_sel_field_store_in_mysql_format(
 	ulint	len)	/* in: length of the data */
 {
 	byte*	ptr;
+	byte*	field_end;
+	byte*	pad_ptr;
 
 	ut_ad(len != UNIV_SQL_NULL);
 
@@ -2274,25 +2278,66 @@ row_sel_field_store_in_mysql_format(
 		}
 
 		ut_ad(templ->mysql_col_len == len);
-	} else if (templ->type == DATA_VARCHAR || templ->type == DATA_VARMYSQL
-					|| templ->type == DATA_BINARY) {
-		/* Store the length of the data to the first two bytes of
-		dest; does not do anything yet because MySQL has
-		no real vars! */
+	} else if (templ->type == DATA_VARCHAR
+	           || templ->type == DATA_VARMYSQL
+		   || templ->type == DATA_BINARY) {
+
+		field_end = dest + templ->mysql_col_len;
+
+		if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) {
+			/* This is a >= 5.0.3 type true VARCHAR. Store the
+			length of the data to the first byte or the first
+			two bytes of dest. */
 		
-		dest = row_mysql_store_var_len(dest, len);
+			dest = row_mysql_store_true_var_len(dest, len,
+						templ->mysql_length_bytes);
+		}
+
+		/* Copy the actual data */
 		ut_memcpy(dest, data, len);
-#if 0
-		/* No real var implemented in MySQL yet! */
-		ut_ad(templ->mysql_col_len >= len + 2);
-#endif
 		
+		/* Pad with trailing spaces. We pad with spaces also the
+		unused end of a >= 5.0.3 true VARCHAR column, just in case
+		MySQL expects its contents to be deterministic. */
+			
+		pad_ptr = dest + len;
+
+		ut_ad(templ->mbminlen <= templ->mbmaxlen);
+
+		/* We handle UCS2 charset strings differently. */
+		if (templ->mbminlen == 2) {
+			/* A space char is two bytes, 0x0020 in UCS2 */
+
+			if (len & 1) {
+				/* A 0x20 has been stripped from the column.
+				Pad it back. */
+				
+				if (pad_ptr < field_end) {
+					*pad_ptr = 0x20;
+					pad_ptr++;
+				}
+			}
+			
+			/* Pad the rest of the string with 0x0020 */
+
+			while (pad_ptr < field_end) {
+				*pad_ptr = 0x00;
+				pad_ptr++;
+				*pad_ptr = 0x20;
+				pad_ptr++;
+			}
+		} else {
+			ut_ad(templ->mbminlen == 1);
+			/* space=0x20 */
+
+			memset(pad_ptr, 0x20, field_end - pad_ptr);
+		}
 	} else if (templ->type == DATA_BLOB) {
 		/* Store a pointer to the BLOB buffer to dest: the BLOB was
 		already copied to the buffer in row_sel_store_mysql_rec */
 
-		row_mysql_store_blob_ref(dest, templ->mysql_col_len,
-							data, len);
+		row_mysql_store_blob_ref(dest, templ->mysql_col_len, data,
+									len);
 	} else if (templ->type == DATA_MYSQL) {
 		memcpy(dest, data, len);
 
@@ -2306,9 +2351,10 @@ row_sel_field_store_in_mysql_format(
 		ut_a(len * templ->mbmaxlen >= templ->mysql_col_len);
 
 		if (templ->mbminlen != templ->mbmaxlen) {
-			/* Pad with spaces.  This undoes the stripping
+			/* Pad with spaces. This undoes the stripping
 			done in row0mysql.ic, function
 			row_mysql_store_col_in_innobase_format(). */
+
 			memset(dest + len, 0x20, templ->mysql_col_len - len);
 		}
 	} else {
@@ -2320,6 +2366,7 @@ row_sel_field_store_in_mysql_format(
 			|| templ->type == DATA_DOUBLE
 			|| templ->type == DATA_DECIMAL);
 		ut_ad(templ->mysql_col_len == len);
+
 		memcpy(dest, data, len);
 	}
 }
@@ -2436,40 +2483,6 @@ row_sel_store_mysql_rec(
 				mysql_rec + templ->mysql_col_offset,
 				templ, data, len);
 
-			if (templ->type == DATA_VARCHAR
-					|| templ->type == DATA_VARMYSQL
-					|| templ->type == DATA_BINARY) {
-				/* Pad with trailing spaces */
-				data = mysql_rec + templ->mysql_col_offset;
-
-				ut_ad(templ->mbminlen <= templ->mbmaxlen);
-				/* Handle UCS2 strings differently. */
-				if (templ->mbminlen == 2) {
-					/* space=0x0020 */
-					ulint	col_len = templ->mysql_col_len;
-
-					ut_a(!(col_len & 1));
-					if (len & 1) {
-						/* A 0x20 has been stripped
-						from the column.
-						Pad it back. */
-						goto pad_0x20;
-					}
-					/* Pad the rest of the string
-					with 0x0020 */
-					while (len < col_len) {
-						data[len++] = 0x00;
-					pad_0x20:
-						data[len++] = 0x20;
-					}
-				} else {
-					ut_ad(templ->mbminlen == 1);
-					/* space=0x20 */
-					memset(data + len, 0x20,
-						templ->mysql_col_len - len);
-				}
-			}
-
 			/* Cleanup */
 			if (extern_field_heap) {
  				mem_heap_free(extern_field_heap);