Applied innodb snapshot ss792 and ss854.

This ChangeSet will be null-merged to mysql-5.1.

Bugs fixed:
- Bug #21638:  InnoDB: possible segfault in page_copy_rec_list_end_no_locks
  If prefix_len is specified, write it to the insert buffer instead of type->len
- bug #10746:  InnoDB: Error: stored_select_lock_type is 0 inside ::start_stmt()!
  INSERT ... SELECT uses LOCK_NONE in innodb_locks_unsafe_for_binlog mode, so do
  not check for LOCK_X or LOCK_S in start_stmt()


innobase/ibuf/ibuf0ibuf.c:
  Applied innodb snapshot ss792 and ss854.
  
  ibuf_entry_build(): Write prefix_len to the insert buffer instead of type->len
  when prefix_len is specified.  Otherwise, btr_page_reorganize() during
  insert buffer merge would fail on ROW_FORMAT=COMPACT tables.  (Bug #21638)
innobase/include/data0type.h:
  Applied innodb snapshot ss792 and ss854.
  
  ibuf_entry_build(): Write prefix_len to the insert buffer instead of type->len
  when prefix_len is specified.  Otherwise, btr_page_reorganize() during
  insert buffer merge would fail on ROW_FORMAT=COMPACT tables.  (Bug #21638)
innobase/include/data0type.ic:
  Applied innodb snapshot ss792 and ss854.
  
  ibuf_entry_build(): Write prefix_len to the insert buffer instead of type->len
  when prefix_len is specified.  Otherwise, btr_page_reorganize() during
  insert buffer merge would fail on ROW_FORMAT=COMPACT tables.  (Bug #21638)
sql/ha_innodb.cc:
  Applied innodb snapshot ss792 and ss854.
  
  ha_innobase::start_stmt(): Remove the check for
  prebuilt->stored_select_lock_type being LOCK_X or LOCK_S.
  This would cause false alarms with INSERT ... SELECT, which would use
  LOCK_NONE in innodb_locks_unsafe_for_binlog mode.  (Bug #10746)

3 files changed, 20 insertions, 12 deletions

diff --git a/innobase/ibuf/ibuf0ibuf.c b/innobase/ibuf/ibuf0ibuf.c
index 9c7d9c5c3da..eb10a88d1d1 100644
--- a/innobase/ibuf/ibuf0ibuf.c
+++ b/innobase/ibuf/ibuf0ibuf.c
@@ -1361,8 +1361,8 @@ ibuf_entry_build(
 				index tree; NOTE that the original entry
 				must be kept because we copy pointers to its
 				fields */
+	dict_index_t*	index,	/* in: non-clustered index */
 	dtuple_t*	entry,	/* in: entry for a non-clustered index */
-	ibool		comp,	/* in: flag: TRUE=compact record format */
 	ulint		space,	/* in: space id */
 	ulint		page_no,/* in: index page number where entry should
 				be inserted */
@@ -1426,13 +1426,13 @@ ibuf_entry_build(
 
 	dfield_set_data(field, buf, 4);
 
-	ut_ad(comp == 0 || comp == 1);
+	ut_ad(index->table->comp <= 1);
 	/* Store the type info in buf2, and add the fields from entry to
 	tuple */
 	buf2 = mem_heap_alloc(heap, n_fields
 					* DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE
-					+ comp);
-	if (comp) {
+					+ index->table->comp);
+	if (index->table->comp) {
 		*buf2++ = 0; /* write the compact format indicator */
 	}
 	for (i = 0; i < n_fields; i++) {
@@ -1445,20 +1445,22 @@ ibuf_entry_build(
 
 		dtype_new_store_for_order_and_null_size(
 				buf2 + i * DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE,
-				dfield_get_type(entry_field));
+				dfield_get_type(entry_field),
+				dict_index_get_nth_field(index, i)
+				->prefix_len);
 	}
 
 	/* Store the type info in buf2 to field 3 of tuple */
 
 	field = dtuple_get_nth_field(tuple, 3);
 
-	if (comp) {
+	if (index->table->comp) {
 		buf2--;
 	}
 
 	dfield_set_data(field, buf2, n_fields
 					* DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE
-					+ comp);
+					+ index->table->comp);
 	/* Set all the types in the new tuple binary */
 
 	dtuple_set_types_binary(tuple, n_fields + 4);
@@ -2589,8 +2591,7 @@ ibuf_insert_low(
 	the first fields and the type information for other fields, and which
 	will be inserted to the insert buffer. */
 
-	ibuf_entry = ibuf_entry_build(entry, index->table->comp,
-						space, page_no, heap);
+	ibuf_entry = ibuf_entry_build(index, entry, space, page_no, heap);
 
 	/* Open a cursor to the insert buffer tree to calculate if we can add
 	the new entry to it without exceeding the free space limit for the
diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h
index 30c0f732c34..abfbc24e336 100644
--- a/innobase/include/data0type.h
+++ b/innobase/include/data0type.h
@@ -366,7 +366,9 @@ dtype_new_store_for_order_and_null_size(
 	byte*		buf,	/* in: buffer for
 				DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE
 				bytes where we store the info */
-	dtype_t*	type);	/* in: type struct */
+	dtype_t*	type,	/* in: type struct */
+	ulint		prefix_len);/* in: prefix length to
+				replace type->len, or 0 */
 /**************************************************************************
 Reads to a type the stored information which determines its alphabetical
 ordering and the storage size of an SQL NULL value. This is the 4.1.x storage
diff --git a/innobase/include/data0type.ic b/innobase/include/data0type.ic
index ad0f02fa63d..032c1d7efd7 100644
--- a/innobase/include/data0type.ic
+++ b/innobase/include/data0type.ic
@@ -230,11 +230,14 @@ dtype_new_store_for_order_and_null_size(
 	byte*		buf,	/* in: buffer for
 				DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE
 				bytes where we store the info */
-	dtype_t*	type)	/* in: type struct */
+	dtype_t*	type,	/* in: type struct */
+	ulint		prefix_len)/* in: prefix length to
+				replace type->len, or 0 */
 {
 #if 6 != DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE
 #error "6 != DATA_NEW_ORDER_NULL_TYPE_BUF_SIZE"
 #endif
+	ulint	len;
 
 	buf[0] = (byte)(type->mtype & 0xFFUL);
 
@@ -249,7 +252,9 @@ dtype_new_store_for_order_and_null_size(
 
 	buf[1] = (byte)(type->prtype & 0xFFUL);
 
-	mach_write_to_2(buf + 2, type->len & 0xFFFFUL);
+	len = prefix_len ? prefix_len : type->len;
+
+	mach_write_to_2(buf + 2, len & 0xFFFFUL);
 
 	ut_ad(dtype_get_charset_coll(type->prtype) < 256);
 	mach_write_to_2(buf + 4, dtype_get_charset_coll(type->prtype));