5.3 merge

13 files changed, 108 insertions, 8 deletions

diff --git a/mysql-test/r/ctype_binary.result b/mysql-test/r/ctype_binary.result
index 4c09fb97e64..3caf41af53b 100644
--- a/mysql-test/r/ctype_binary.result
+++ b/mysql-test/r/ctype_binary.result
@@ -740,7 +740,7 @@ create table t1 as select concat(uncompressed_length('')) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varbinary(10) NOT NULL DEFAULT ''
+  `c1` varbinary(10) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select concat(connection_id()) as c1;
diff --git a/mysql-test/r/ctype_cp1251.result b/mysql-test/r/ctype_cp1251.result
index 723f7534259..4b8a26265e3 100644
--- a/mysql-test/r/ctype_cp1251.result
+++ b/mysql-test/r/ctype_cp1251.result
@@ -1132,7 +1132,7 @@ create table t1 as select concat(uncompressed_length('')) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(10) CHARACTER SET cp1251 NOT NULL DEFAULT ''
+  `c1` varchar(10) CHARACTER SET cp1251 DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select concat(connection_id()) as c1;
diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result
index 52b30808f34..9ba82279401 100644
--- a/mysql-test/r/ctype_latin1.result
+++ b/mysql-test/r/ctype_latin1.result
@@ -1212,7 +1212,7 @@ create table t1 as select concat(uncompressed_length('')) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(10) NOT NULL DEFAULT ''
+  `c1` varchar(10) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select concat(connection_id()) as c1;
diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index 52ce5efa3e4..29f409075a3 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -2121,7 +2121,7 @@ create table t1 as select concat(uncompressed_length('')) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(10) CHARACTER SET ucs2 NOT NULL DEFAULT ''
+  `c1` varchar(10) CHARACTER SET ucs2 DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select concat(connection_id()) as c1;
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 802006c4c66..fa76cc54c39 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -2951,7 +2951,7 @@ create table t1 as select concat(uncompressed_length('')) as c1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(10) CHARACTER SET utf8 NOT NULL DEFAULT ''
+  `c1` varchar(10) CHARACTER SET utf8 DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
 create table t1 as select concat(connection_id()) as c1;
diff --git a/mysql-test/r/func_compress.result b/mysql-test/r/func_compress.result
index 011dec4d555..22b75de8c60 100644
--- a/mysql-test/r/func_compress.result
+++ b/mysql-test/r/func_compress.result
@@ -127,3 +127,23 @@ NULL	825307441
 EXPLAIN EXTENDED SELECT * FROM (SELECT UNCOMPRESSED_LENGTH(c1) FROM t1) AS s;
 DROP TABLE t1;
 End of 5.0 tests
+#
+# Start of 5.3 tests
+#
+#
+# MDEV-5783 Assertion `0' failed in make_sortkey(SORTPARAM*, uchar*, uchar*) on ORDER BY HEX(UNCOMPRESSED_LENGTH(pk))
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+SELECT UNCOMPRESSED_LENGTH(pk) FROM t1;
+UNCOMPRESSED_LENGTH(pk)
+NULL
+NULL
+Warnings:
+Warning	1259	ZLIB: Input data corrupted
+Warning	1259	ZLIB: Input data corrupted
+SELECT * FROM t1 ORDER BY HEX(UNCOMPRESSED_LENGTH(pk));
+DROP TABLE t1;
+#
+# End of 5.3 tests
+#
diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result
index 09271e21c66..a78cc1c3a9e 100644
--- a/mysql-test/r/subselect_mat.result
+++ b/mysql-test/r/subselect_mat.result
@@ -2055,6 +2055,20 @@ EXECUTE stmt;
 a
 DROP TABLE t1, t2;
 DROP VIEW v2;
+#
+# MDEV-5811: Server crashes in best_access_path with materialization+semijoin and big_tables=ON
+#
+SET @tmp_mdev5811= @@big_tables;
+SET big_tables = ON;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+SELECT * FROM t1 AS t1_1, t1 AS t1_2 
+WHERE ( t1_1.a, t1_2.a ) IN ( SELECT MAX(b), MIN(b) FROM t2 );
+a	a
+DROP TABLE t1,t2;
+SET big_tables=@tmp_mdev5811;
 # End of 5.3 tests
 #
 # MDEV-5056: Wrong result (extra rows) with materialization+semijoin, IN subqueries
diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result
index ba34543c2e3..c9c2c735428 100644
--- a/mysql-test/r/subselect_sj_mat.result
+++ b/mysql-test/r/subselect_sj_mat.result
@@ -2095,6 +2095,20 @@ EXECUTE stmt;
 a
 DROP TABLE t1, t2;
 DROP VIEW v2;
+#
+# MDEV-5811: Server crashes in best_access_path with materialization+semijoin and big_tables=ON
+#
+SET @tmp_mdev5811= @@big_tables;
+SET big_tables = ON;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+SELECT * FROM t1 AS t1_1, t1 AS t1_2 
+WHERE ( t1_1.a, t1_2.a ) IN ( SELECT MAX(b), MIN(b) FROM t2 );
+a	a
+DROP TABLE t1,t2;
+SET big_tables=@tmp_mdev5811;
 # End of 5.3 tests
 #
 # MDEV-5056: Wrong result (extra rows) with materialization+semijoin, IN subqueries
diff --git a/mysql-test/t/func_compress.test b/mysql-test/t/func_compress.test
index 207f3a436d0..eaed0c88fe1 100644
--- a/mysql-test/t/func_compress.test
+++ b/mysql-test/t/func_compress.test
@@ -115,3 +115,24 @@ DROP TABLE t1;
 set @@global.max_allowed_packet=default;
 --enable_result_log
 --enable_query_log
+
+
+--echo #
+--echo # Start of 5.3 tests
+--echo #
+
+--echo #
+--echo # MDEV-5783 Assertion `0' failed in make_sortkey(SORTPARAM*, uchar*, uchar*) on ORDER BY HEX(UNCOMPRESSED_LENGTH(pk))
+--echo #
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+SELECT UNCOMPRESSED_LENGTH(pk) FROM t1;
+# ORDER is not strict, so disable results
+--disable_result_log
+SELECT * FROM t1 ORDER BY HEX(UNCOMPRESSED_LENGTH(pk));
+--enable_result_log
+DROP TABLE t1;
+
+--echo #
+--echo # End of 5.3 tests
+--echo #
diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test
index 52a73d24822..91b69a6a09c 100644
--- a/mysql-test/t/subselect_sj_mat.test
+++ b/mysql-test/t/subselect_sj_mat.test
@@ -1745,6 +1745,24 @@ EXECUTE stmt;
 DROP TABLE t1, t2;
 DROP VIEW v2;
 
+--echo #
+--echo # MDEV-5811: Server crashes in best_access_path with materialization+semijoin and big_tables=ON
+--echo #
+SET @tmp_mdev5811= @@big_tables;
+SET big_tables = ON;
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+
+CREATE TABLE t2 (b INT);
+INSERT INTO t2 VALUES (3),(4);
+
+SELECT * FROM t1 AS t1_1, t1 AS t1_2 
+  WHERE ( t1_1.a, t1_2.a ) IN ( SELECT MAX(b), MIN(b) FROM t2 );
+
+DROP TABLE t1,t2;
+SET big_tables=@tmp_mdev5811;
+
 --echo # End of 5.3 tests
 
 
diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
index 1731fcf7e99..4579a6949c7 100644
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@ -943,7 +943,7 @@ class Item_func_uncompressed_length : public Item_int_func
 public:
   Item_func_uncompressed_length(Item *a):Item_int_func(a){}
   const char *func_name() const{return "uncompressed_length";}
-  void fix_length_and_dec() { max_length=10; }
+  void fix_length_and_dec() { max_length=10; maybe_null= true; }
   longlong val_int();
 };
 
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 66bd287d86a..103c79fa0d3 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -13636,7 +13636,7 @@ int QUICK_GROUP_MIN_MAX_SELECT::next_min()
       }
       else if (result == HA_ERR_KEY_NOT_FOUND || result == HA_ERR_END_OF_FILE)
         result= 0; /* There is a result in any case. */
-      my_afree(tmp_key_buff);
+      my_afree(key_buf);
     }
   }
 
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 721c774970d..50d700b56b4 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -15469,7 +15469,20 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields,
     keyinfo->key_length= 0;  // Will compute the sum of the parts below.
     keyinfo->name= (char*) "distinct_key";
     keyinfo->algorithm= HA_KEY_ALG_UNDEF;
-    keyinfo->rec_per_key=0;
+    /*
+      Needed by non-merged semi-joins: SJ-Materialized table must have a valid 
+      rec_per_key array, because it participates in join optimization. Since
+      the table has no data, the only statistics we can provide is "unknown",
+      i.e. zero values.
+
+      (For table record count, we calculate and set JOIN_TAB::found_records,
+       see get_delayed_table_estimates()).
+    */
+    size_t rpk_size= keyinfo->key_parts* sizeof(keyinfo->rec_per_key[0]);
+    if (!(keyinfo->rec_per_key= (ulong*) alloc_root(&table->mem_root, 
+                                                    rpk_size)))
+      goto err;
+    bzero(keyinfo->rec_per_key, rpk_size);
 
     /*
       Create an extra field to hold NULL bits so that unique indexes on