MDEV-19705: Assertion `tmp >= 0' failed in best_access_path

The reason for hitting the assert is that rec_per_key estimates have some garbage value.
So the solution to fix this would be for long unique keys to use use rec_per_key for only 1 keypart,
that means rec_per_key[0] would have the estimate.

3 files changed, 32 insertions, 1 deletions

diff --git a/mysql-test/main/long_unique.result b/mysql-test/main/long_unique.result
index 8ea6d36c321..a4955b3e7b5 100644
--- a/mysql-test/main/long_unique.result
+++ b/mysql-test/main/long_unique.result
@@ -1462,4 +1462,19 @@ t1	CREATE TABLE `t1` (
   KEY `pk` (`pk`,`a`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 drop table t1;
+#
+# MDEV-19705: Assertion `tmp >= 0' failed in best_access_path
+#
+CREATE TABLE t1 (d varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('a'),('q');
+CREATE TABLE t2 (f varchar(10), a2 datetime, b int, a1 varchar(1024), pk int NOT NULL, PRIMARY KEY (pk), UNIQUE KEY (f,a1,a2), KEY f2 (f(4),a2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES ('aaa','1985-09-06',-163,'s',1),('bbb','1995-01-05',3,'pucaz',2),('ccc','0000-00-00',NULL,'help',3),('ddd',NULL,618,'v',4),('eee','1995-12-20',410,'m',5),('ffq','1976-06-12 20:02:56',NULL,'POKNC',6),('dddd','0000-00-00',-328,'hgsu',7);
+explain
+SELECT t2.b FROM t1 JOIN t2 ON t1.d = t2.f WHERE t2.pk >= 20;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	range	PRIMARY,f,f2	PRIMARY	4	NULL	1	Using index condition
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+SELECT t2.b FROM t1 JOIN t2 ON t1.d = t2.f WHERE t2.pk >= 20;
+b
+drop table t1,t2;
 set @@GLOBAL.max_allowed_packet= @allowed_packet;
diff --git a/mysql-test/main/long_unique.test b/mysql-test/main/long_unique.test
index a6bc68f54dc..c0bd77ca5c9 100644
--- a/mysql-test/main/long_unique.test
+++ b/mysql-test/main/long_unique.test
@@ -542,4 +542,18 @@ alter table t1 modify a varchar(1000);
 show create table t1;
 drop table t1;
 
+--echo #
+--echo # MDEV-19705: Assertion `tmp >= 0' failed in best_access_path
+--echo #
+
+CREATE TABLE t1 (d varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('a'),('q');
+ 
+CREATE TABLE t2 (f varchar(10), a2 datetime, b int, a1 varchar(1024), pk int NOT NULL, PRIMARY KEY (pk), UNIQUE KEY (f,a1,a2), KEY f2 (f(4),a2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES ('aaa','1985-09-06',-163,'s',1),('bbb','1995-01-05',3,'pucaz',2),('ccc','0000-00-00',NULL,'help',3),('ddd',NULL,618,'v',4),('eee','1995-12-20',410,'m',5),('ffq','1976-06-12 20:02:56',NULL,'POKNC',6),('dddd','0000-00-00',-328,'hgsu',7); 
+explain
+SELECT t2.b FROM t1 JOIN t2 ON t1.d = t2.f WHERE t2.pk >= 20;
+SELECT t2.b FROM t1 JOIN t2 ON t1.d = t2.f WHERE t2.pk >= 20;
+drop table t1,t2;
+
 set @@GLOBAL.max_allowed_packet= @allowed_packet;
diff --git a/sql/table.cc b/sql/table.cc
index 1ab4df0f7cf..da0c4be41b0 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -800,7 +800,8 @@ static bool create_key_infos(const uchar *strpos, const uchar *frm_image_end,
     {
       if (strpos + (new_frm_ver >= 1 ? 9 : 7) >= frm_image_end)
         return 1;
-      *rec_per_key++=0;
+      if (!(keyinfo->algorithm == HA_KEY_ALG_LONG_HASH))
+        *rec_per_key++=0;
       key_part->fieldnr=	(uint16) (uint2korr(strpos) & FIELD_NR_MASK);
       key_part->offset= (uint) uint2korr(strpos+2)-1;
       key_part->key_type=	(uint) uint2korr(strpos+5);
@@ -828,6 +829,7 @@ static bool create_key_infos(const uchar *strpos, const uchar *frm_image_end,
     {
       keyinfo->key_length= HA_HASH_KEY_LENGTH_WITHOUT_NULL;
       key_part++; // reserved for the hash value
+      *rec_per_key++=0;
     }
 
     /*