MDEV-10759 Fix Aria to support 2-byte collation IDs

- Used same fix as for MyISAM: High level collation byte stored in unused
  bit_end position.
- Moved language from header to base_info
- Removed unused bit_end part in HA_KEY_SEG

14 files changed, 72 insertions, 26 deletions

diff --git a/include/maria.h b/include/maria.h
index bdd53f3d183..1305b6444c0 100644
--- a/include/maria.h
+++ b/include/maria.h
@@ -149,7 +149,7 @@ typedef struct st_maria_create_info
   uint null_bytes;
   uint old_options;
   enum data_file_type org_data_file_type;
-  uint8 language;
+  uint16 language;
   my_bool with_auto_increment, transactional;
 } MARIA_CREATE_INFO;
 
diff --git a/include/my_compare.h b/include/my_compare.h
index 0db22b593f4..3440d9ef920 100644
--- a/include/my_compare.h
+++ b/include/my_compare.h
@@ -57,7 +57,7 @@ typedef struct st_HA_KEYSEG		/* Key-portion */
   uint16 language;
   uint8  type;				/* Type of key (for sort) */
   uint8  null_bit;			/* bitmask to test for NULL */
-  uint8  bit_start,bit_end;		/* if bit field */
+  uint8  bit_start;
   uint8  bit_length;                    /* Length of bit part */
 } HA_KEYSEG;
 
diff --git a/mysql-test/suite/maria/collations.result b/mysql-test/suite/maria/collations.result
new file mode 100644
index 00000000000..86f7117c378
--- /dev/null
+++ b/mysql-test/suite/maria/collations.result
@@ -0,0 +1,25 @@
+DROP TABLE IF EXISTS t1;
+Warnings:
+Note	1051	Unknown table 'test.t1'
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_croatian_ci, KEY(a)) ENGINE=ARIA;
+INSERT INTO t1 VALUES ('na'),('nj'),('nz'),('Z');
+explain SELECT a FROM t1 ORDER BY a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	a	33	NULL	4	Using index
+SELECT a FROM t1 ORDER BY a;
+a
+na
+nz
+nj
+Z
+ALTER TABLE t1 engine=myisam;
+explain SELECT a FROM t1 ORDER BY a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	a	33	NULL	4	Using index
+SELECT a FROM t1 ORDER BY a;
+a
+na
+nz
+nj
+Z
+drop table t1;
diff --git a/mysql-test/suite/maria/collations.test b/mysql-test/suite/maria/collations.test
new file mode 100644
index 00000000000..fe93e1913a7
--- /dev/null
+++ b/mysql-test/suite/maria/collations.test
@@ -0,0 +1,14 @@
+#
+# Test 2-byte collations
+#
+
+DROP TABLE IF EXISTS t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_croatian_ci, KEY(a)) ENGINE=ARIA;
+INSERT INTO t1 VALUES ('na'),('nj'),('nz'),('Z');
+explain SELECT a FROM t1 ORDER BY a;
+SELECT a FROM t1 ORDER BY a;
+ALTER TABLE t1 engine=myisam;
+explain SELECT a FROM t1 ORDER BY a;
+SELECT a FROM t1 ORDER BY a;
+drop table t1;
diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index c6d457eeea8..c1a99370d80 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -549,8 +549,7 @@ static int table2maria(TABLE *table_arg, data_file_type row_type,
       keydef[i].seg[j].type= (int) type;
       keydef[i].seg[j].start= pos->key_part[j].offset;
       keydef[i].seg[j].length= pos->key_part[j].length;
-      keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_end=
-        keydef[i].seg[j].bit_length= 0;
+      keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_length= 0;
       keydef[i].seg[j].bit_pos= 0;
       keydef[i].seg[j].language= field->charset()->number;
 
diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c
index 5035df26b18..8424817bc6b 100644
--- a/storage/maria/ma_check.c
+++ b/storage/maria/ma_check.c
@@ -6103,7 +6103,7 @@ int maria_recreate_table(HA_CHECK *param, MARIA_HA **org_info, char *filename)
   create_info.data_file_length=file_length;
   create_info.auto_increment=share.state.auto_increment;
   create_info.language = (param->language ? param->language :
-			  share.state.header.language);
+			  share.base.language);
   create_info.key_file_length=  status_info.key_file_length;
   create_info.org_data_file_type= ((enum data_file_type)
                                    share.state.header.org_data_file_type);
diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c
index f160499a94e..0680b5d568e 100644
--- a/storage/maria/ma_create.c
+++ b/storage/maria/ma_create.c
@@ -725,8 +725,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
   mi_int2store(share.state.header.base_pos,base_pos);
   share.state.header.data_file_type= share.data_file_type= datafile_type;
   share.state.header.org_data_file_type= org_datafile_type;
-  share.state.header.language= (ci->language ?
-				ci->language : default_charset_info->number);
+  share.state.header.not_used= 0;
 
   share.state.dellink = HA_OFFSET_ERROR;
   share.state.first_bitmap_with_space= 0;
@@ -739,6 +738,8 @@ int maria_create(const char *name, enum data_file_type datafile_type,
   share.options=options;
   share.base.rec_reflength=pointer;
   share.base.block_size= maria_block_size;
+  share.base.language= (ci->language ? ci->language :
+                        default_charset_info->number);
 
   /*
     Get estimate for index file length (this may be wrong for FT keys)
@@ -937,7 +938,6 @@ int maria_create(const char *name, enum data_file_type datafile_type,
       sseg.language= 7;                         /* Binary */
       sseg.null_bit=0;
       sseg.bit_start=0;
-      sseg.bit_end=0;
       sseg.bit_length= 0;
       sseg.bit_pos= 0;
       sseg.length=SPLEN;
diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c
index 42861e92ed4..4e97c6b43b9 100644
--- a/storage/maria/ma_open.c
+++ b/storage/maria/ma_open.c
@@ -276,6 +276,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
   uint i,j,len,errpos,head_length,base_pos,keys, realpath_err,
     key_parts,unique_key_parts,fulltext_keys,uniques;
   uint internal_table= MY_TEST(open_flags & HA_OPEN_INTERNAL_TABLE);
+  uint file_version;
   size_t info_length;
   char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN],
        data_name[FN_REFLEN];
@@ -335,8 +336,8 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
     }
     share->mode=open_mode;
     errpos= 1;
-    if (mysql_file_pread(kfile,share->state.header.file_version, head_length, 0,
-                 MYF(MY_NABP)))
+    if (mysql_file_pread(kfile,share->state.header.file_version, head_length,
+                         0, MYF(MY_NABP)))
     {
       my_errno= HA_ERR_NOT_A_TABLE;
       goto err;
@@ -429,6 +430,14 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
 			    len,MARIA_BASE_INFO_SIZE));
     }
     disk_pos= _ma_base_info_read(disk_cache + base_pos, &share->base);
+    /*
+      Check if old version of Aria file. Version 0 has language
+      stored in header.not_used
+    */
+    file_version= (share->state.header.not_used == 0);
+    if (file_version == 0)
+      share->base.language= share->state.header.not_used;
+    
     share->state.state_length=base_pos;
     /* For newly opened tables we reset the error-has-been-printed flag */
     share->state.changed&= ~STATE_CRASHED_PRINTED;
@@ -1581,7 +1590,7 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base)
   mi_int2store(ptr,base->null_bytes);                   ptr+= 2;
   mi_int2store(ptr,base->original_null_bytes);	        ptr+= 2;
   mi_int2store(ptr,base->field_offsets);	        ptr+= 2;
-  mi_int2store(ptr,0);				        ptr+= 2; /* reserved */
+  mi_int2store(ptr,base->language);		        ptr+= 2;
   mi_int2store(ptr,base->block_size);	        	ptr+= 2;
   *ptr++= base->rec_reflength;
   *ptr++= base->key_reflength;
@@ -1624,7 +1633,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base)
   base->null_bytes= mi_uint2korr(ptr);			ptr+= 2;
   base->original_null_bytes= mi_uint2korr(ptr);		ptr+= 2;
   base->field_offsets= mi_uint2korr(ptr);		ptr+= 2;
-                                                        ptr+= 2;
+  base->language= mi_uint2korr(ptr);		        ptr+= 2;
   base->block_size= mi_uint2korr(ptr);			ptr+= 2;
 
   base->rec_reflength= *ptr++;
@@ -1689,10 +1698,10 @@ my_bool _ma_keyseg_write(File file, const HA_KEYSEG *keyseg)
   ulong pos;
 
   *ptr++= keyseg->type;
-  *ptr++= keyseg->language;
+  *ptr++= keyseg->language & 0xFF; /* Collation ID, low byte */
   *ptr++= keyseg->null_bit;
   *ptr++= keyseg->bit_start;
-  *ptr++= keyseg->bit_end;
+  *ptr++= keyseg->language >> 8; /* Collation ID, high byte */
   *ptr++= keyseg->bit_length;
   mi_int2store(ptr,keyseg->flag);	ptr+= 2;
   mi_int2store(ptr,keyseg->length);	ptr+= 2;
@@ -1711,7 +1720,7 @@ uchar *_ma_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg)
    keyseg->language	= *ptr++;
    keyseg->null_bit	= *ptr++;
    keyseg->bit_start	= *ptr++;
-   keyseg->bit_end	= *ptr++;
+   keyseg->language	+= ((uint16) (*ptr++)) << 8;
    keyseg->bit_length   = *ptr++;
    keyseg->flag		= mi_uint2korr(ptr);  ptr+= 2;
    keyseg->length	= mi_uint2korr(ptr);  ptr+= 2;
diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c
index c9d38400bc4..0c1c56dfa94 100644
--- a/storage/maria/maria_chk.c
+++ b/storage/maria/maria_chk.c
@@ -1120,7 +1120,7 @@ static int maria_chk(HA_CHECK *param, char *filename)
        maria_test_if_almost_full(info) ||
        info->s->state.header.file_version[3] != maria_file_magic[3] ||
        (set_collation &&
-        set_collation->number != share->state.header.language)))
+        set_collation->number != share->base.language)))
   {
     if (set_collation)
       param->language= set_collation->number;
@@ -1507,8 +1507,8 @@ static void descript(HA_CHECK *param, register MARIA_HA *info, char *name)
   printf("Crashsafe:           %s\n",
          share->base.born_transactional ? "yes" : "no");
   printf("Character set:       %s (%d)\n",
-	 get_charset_name(share->state.header.language),
-	 share->state.header.language);
+	 get_charset_name(share->base.language),
+         (int) share->base.language);
 
   if (param->testflag & T_VERBOSE)
   {
diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h
index a4fac8c088a..d216384a2aa 100644
--- a/storage/maria/maria_def.h
+++ b/storage/maria/maria_def.h
@@ -139,7 +139,7 @@ typedef struct st_maria_state_info
     uchar unique_key_parts[2];		/* Key parts + unique parts */
     uchar keys;				/* number of keys in file */
     uchar uniques;			/* number of UNIQUE definitions */
-    uchar language;			/* Language for indexes */
+    uchar not_used;			/* Language for indexes */
     uchar fulltext_keys;
     uchar data_file_type;
     /* Used by mariapack to store the original data_file_type */
@@ -209,6 +209,7 @@ typedef struct st_maria_state_info
 } MARIA_STATE_INFO;
 
 
+/* Number of bytes written be _ma_state_info_write_sub() */
 #define MARIA_STATE_INFO_SIZE	\
   (24 + 2 + LSN_STORE_SIZE*3 + 4 + 11*8 + 4*4 + 8 + 3*4 + 5*8)
 #define MARIA_FILE_OPEN_COUNT_OFFSET 0
@@ -291,6 +292,8 @@ typedef struct st_ma_base_info
   uint extra_rec_buff_size;
   /* Tuning flags that can be ignored by older Maria versions */
   uint extra_options;
+  /* default language, not really used but displayed by maria_chk */
+  uint language;
 
   /* The following are from the header */
   uint key_parts, all_key_parts;
@@ -916,7 +919,6 @@ extern mysql_mutex_t THR_LOCK_maria;
 #define MARIA_SMALL_BLOB_BUFFER 1024
 #define MARIA_MAX_CONTROL_FILE_LOCK_RETRY 30     /* Retry this many times */
 
-
 /* Some extern variables */
 extern LIST *maria_open_list;
 extern uchar maria_file_magic[], maria_pack_file_magic[];
diff --git a/storage/myisam/ft_static.c b/storage/myisam/ft_static.c
index aa8adba88a2..92a0621fd9f 100644
--- a/storage/myisam/ft_static.c
+++ b/storage/myisam/ft_static.c
@@ -34,7 +34,7 @@ const HA_KEYSEG ft_keysegs[FT_SEGS]= {
   63,                                           /* language (will be overwritten) */
   HA_KEYTYPE_VARTEXT2,                          /* type */
   0,                                            /* null_bit */
-  2, 0, 0                                       /* bit_start, bit_end, bit_length */
+  2, 0                                          /* bit_start, bit_length */
 },
 {
   /*
@@ -42,7 +42,7 @@ const HA_KEYSEG ft_keysegs[FT_SEGS]= {
       be packed in any way, otherwise w_search() won't be able to
       update key entry 'in vivo'
     */
-  0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0, 0
+  0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0
 }
 };
 
diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc
index 40a27de5cf9..7d518320d50 100644
--- a/storage/myisam/ha_myisam.cc
+++ b/storage/myisam/ha_myisam.cc
@@ -279,8 +279,7 @@ int table2myisam(TABLE *table_arg, MI_KEYDEF **keydef_out,
       keydef[i].seg[j].type= (int) type;
       keydef[i].seg[j].start= pos->key_part[j].offset;
       keydef[i].seg[j].length= pos->key_part[j].length;
-      keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_end=
-        keydef[i].seg[j].bit_length= 0;
+      keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_length= 0;
       keydef[i].seg[j].bit_pos= 0;
       keydef[i].seg[j].language= field->charset_for_protocol()->number;
 
diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c
index 88b9da6f8a9..c781538dc0f 100644
--- a/storage/myisam/mi_create.c
+++ b/storage/myisam/mi_create.c
@@ -729,7 +729,6 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
       sseg.language= 7;                         /* Binary */
       sseg.null_bit=0;
       sseg.bit_start=0;
-      sseg.bit_end=0;
       sseg.bit_length= 0;
       sseg.bit_pos= 0;
       sseg.length=SPLEN;
diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c
index b8cbefe6ac2..2d794f611d7 100644
--- a/storage/myisam/mi_open.c
+++ b/storage/myisam/mi_open.c
@@ -1183,7 +1183,6 @@ uchar *mi_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg)
    keyseg->length	= mi_uint2korr(ptr);  ptr +=2;
    keyseg->start	= mi_uint4korr(ptr);  ptr +=4;
    keyseg->null_pos	= mi_uint4korr(ptr);  ptr +=4;
-   keyseg->bit_end= 0;
    keyseg->charset=0;				/* Will be filled in later */
    if (keyseg->null_bit)
      /* We adjust bit_pos if null_bit is last in the byte */