Fixed MDEV-14326 engine ARIA with row_format=FIXED is broken

The problem was that max_size was acciently set to 1 in some
cases.

Other things:
- Adjust max_rows if min_rows > max_rows.
- Removed not used variable varchar_length
- Adjusted max_pack_length (safety fix)

3 files changed, 184 insertions, 8 deletions

diff --git a/mysql-test/suite/maria/max_length.result b/mysql-test/suite/maria/max_length.result
index 049b92eafe5..177810f32a3 100644
--- a/mysql-test/suite/maria/max_length.result
+++ b/mysql-test/suite/maria/max_length.result
@@ -54,3 +54,108 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	warning	Datafile is almost full, 268230656 of 268320768 used
 test.t1	check	status	OK
 drop table t1,t2;
+create table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED min_rows=1000000;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 MIN_ROWS=1000000 PAGE_CHECKSUM=1 ROW_FORMAT=FIXED
+insert into t1 select seq,seq from seq_1_to_100000;
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 ROW_FORMAT=FIXED
+insert into t1 select seq,seq from seq_1_to_100000;
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE TRANSACTIONAL=0;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 ROW_FORMAT=PAGE TRANSACTIONAL=0
+insert into t1 select seq,seq from seq_1_to_100000;
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED MAX_ROWS=10;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 MAX_ROWS=10 PAGE_CHECKSUM=1 ROW_FORMAT=FIXED
+insert into t1 select seq,seq from seq_1_to_100000;
+ERROR HY000: The table 't1' is full
+select count(*) from t1;
+count(*)
+65535
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=DYNAMIC MAX_ROWS=10;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 MAX_ROWS=10 PAGE_CHECKSUM=1 ROW_FORMAT=DYNAMIC
+insert into t1 select seq,seq from seq_1_to_100000;
+ERROR HY000: The table 't1' is full
+select count(*) from t1;
+count(*)
+3276
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	warning	Datafile is almost full, 65520 of 65535 used
+test.t1	check	status	OK
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE MAX_ROWS=10;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 MAX_ROWS=10 PAGE_CHECKSUM=1 ROW_FORMAT=PAGE
+insert into t1 select seq,seq from seq_1_to_100000;
+select count(*) from t1;
+count(*)
+100000
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+drop table t1;
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE MAX_ROWS=10;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(10) unsigned DEFAULT NULL,
+  `c2` char(80) DEFAULT NULL
+) ENGINE=Aria DEFAULT CHARSET=latin1 MAX_ROWS=10 PAGE_CHECKSUM=1 ROW_FORMAT=PAGE
+insert into t1 select seq,seq from seq_1_to_10000000;
+ERROR HY000: The table 't1' is full
+select count(*) from t1;
+count(*)
+6189940
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	warning	Datafile is almost full, 268320768 of 268320768 used
+test.t1	check	status	OK
+drop table t1;
diff --git a/mysql-test/suite/maria/max_length.test b/mysql-test/suite/maria/max_length.test
index 68ad1e22aa9..2be3da8e1b0 100644
--- a/mysql-test/suite/maria/max_length.test
+++ b/mysql-test/suite/maria/max_length.test
@@ -2,6 +2,7 @@
 # This test will use around 1.3G of disk space!
 
 --source include/have_maria.inc
+--source include/have_sequence.inc
 --source include/big_test.inc
 
 drop table if exists t1,t2;
@@ -50,3 +51,74 @@ insert into t1 (v,b) select v,b from t2;
 check table t1;
 
 drop table t1,t2;
+
+#
+# Check that we don't get table-is-full
+#
+
+create table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED min_rows=1000000;
+show create table t1;
+insert into t1 select seq,seq from seq_1_to_100000;
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED;
+show create table t1;
+insert into t1 select seq,seq from seq_1_to_100000;
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE TRANSACTIONAL=0;
+show create table t1;
+insert into t1 select seq,seq from seq_1_to_100000;
+
+#
+# For these we should get table is full error
+#
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=FIXED MAX_ROWS=10;
+show create table t1;
+--error ER_RECORD_FILE_FULL
+insert into t1 select seq,seq from seq_1_to_100000;
+select count(*) from t1;
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=DYNAMIC MAX_ROWS=10;
+show create table t1;
+--error ER_RECORD_FILE_FULL
+insert into t1 select seq,seq from seq_1_to_100000;
+select count(*) from t1;
+check table t1;
+
+# PAGE uses 3 byte pointers as minimum, which can handle up to 200M files
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE MAX_ROWS=10;
+show create table t1;
+insert into t1 select seq,seq from seq_1_to_100000;
+select count(*) from t1;
+check table t1;
+drop table t1;
+
+create or replace table t1 (
+c1 int unsigned,
+c2 char(80)
+) Engine=ARIA ROW_FORMAT=PAGE MAX_ROWS=10;
+show create table t1;
+--error ER_RECORD_FILE_FULL
+insert into t1 select seq,seq from seq_1_to_10000000;
+select count(*) from t1;
+check table t1;
+drop table t1;
diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c
index 2c67195b3f8..94c4c250bef 100644
--- a/storage/maria/ma_create.c
+++ b/storage/maria/ma_create.c
@@ -70,7 +70,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
   myf create_flag;
   uint length,max_key_length,packed,pack_bytes,pointer,real_length_diff,
        key_length,info_length,key_segs,options,min_key_length,
-       base_pos,long_varchar_count,varchar_length,
+       base_pos,long_varchar_count,
        unique_key_parts,fulltext_keys,offset, not_block_record_extra_length;
   uint max_field_lengths, extra_header_size, column_nr;
   uint internal_table= flags & HA_CREATE_INTERNAL_TABLE;
@@ -144,9 +144,6 @@ int maria_create(const char *name, enum data_file_type datafile_type,
       datafile_type= BLOCK_RECORD;
   }
 
-  if (ci->reloc_rows > ci->max_rows)
-    ci->reloc_rows=ci->max_rows;		/* Check if wrong parameter */
-
   if (!(rec_per_key_part=
 	(double*) my_malloc((keys + uniques)*HA_MAX_KEY_SEG*sizeof(double) +
                             (keys + uniques)*HA_MAX_KEY_SEG*sizeof(ulong) +
@@ -160,7 +157,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
 
 
   /* Start by checking fields and field-types used */
-  varchar_length=long_varchar_count=packed= not_block_record_extra_length=
+  long_varchar_count=packed= not_block_record_extra_length=
     pack_reclength= max_field_lengths= 0;
   reclength= min_pack_length= ci->null_bytes;
   forced_packed= 0;
@@ -232,7 +229,6 @@ int maria_create(const char *name, enum data_file_type datafile_type,
       }
       else if (type == FIELD_VARCHAR)
       {
-	varchar_length+= column->length-1; /* Used for min_pack_length */
 	pack_reclength++;
         not_block_record_extra_length++;
         max_field_lengths++;
@@ -368,6 +364,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
                                 pack_bytes);
   if (!ci->data_file_length && ci->max_rows)
   {
+    set_if_bigger(ci->max_rows, ci->reloc_rows);
     if (pack_reclength == INT_MAX32 ||
              (~(ulonglong) 0)/ci->max_rows < (ulonglong) pack_reclength)
       ci->data_file_length= ~(ulonglong) 0;
@@ -401,13 +398,14 @@ int maria_create(const char *name, enum data_file_type datafile_type,
       else
         ci->max_rows= data_file_length / (min_pack_length +
                                           extra_header_size +
-                                          DIR_ENTRY_SIZE)+1;
+                                          DIR_ENTRY_SIZE);
     }
     else
       ci->max_rows=(ha_rows) (ci->data_file_length/(min_pack_length +
                                                     ((options &
                                                       HA_OPTION_PACK_RECORD) ?
-                                                     3 : 0)))+1;
+                                                     3 : 0)));
+    set_if_smaller(ci->reloc_rows, ci->max_rows);
   }
   max_rows= (ulonglong) ci->max_rows;
   if (datafile_type == BLOCK_RECORD)
@@ -800,6 +798,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     share.state.state.data_file_length= maria_block_size;
     /* Add length of packed fields + length */
     share.base.pack_reclength+= share.base.max_field_lengths+3;
+    share.base.max_pack_length= share.base.pack_reclength;
 
     /* Adjust max_pack_length, to be used if we have short rows */
     if (share.base.max_pack_length < maria_block_size)