diff options
27 files changed, 651 insertions, 326 deletions
diff --git a/include/my_global.h b/include/my_global.h index 30e8909ed13..1b59e1b1a36 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -847,7 +847,8 @@ typedef long my_ptrdiff_t; typedef long long my_ptrdiff_t; #endif -#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) +#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) +#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) #define ALIGN_MAX_UNIT (sizeof(double)) /* Size to make adressable obj. */ diff --git a/include/myisam.h b/include/myisam.h index d5484b1eac6..b3b953b6e99 100644 --- a/include/myisam.h +++ b/include/myisam.h @@ -356,8 +356,10 @@ typedef struct st_mi_sort_param ulonglong notnull[HA_MAX_KEY_SEG+1]; my_off_t pos,max_pos,filepos,start_recpos; - uint key, key_length,real_key_length,sortbuff_size; - uint maxbuffers, keys, find_length, sort_keys_length; + uint key, key_length,real_key_length; + uint maxbuffers, find_length; + ulonglong sortbuff_size; + ha_rows keys; my_bool fix_datafile, master; my_bool calc_checksum; /* calculate table checksum */ @@ -366,10 +368,10 @@ typedef struct st_mi_sort_param int (*key_write)(struct st_mi_sort_param *, const void *); void (*lock_in_memory)(HA_CHECK *); int (*write_keys)(struct st_mi_sort_param *, register uchar **, - uint , struct st_buffpek *, IO_CACHE *); - uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); + ulonglong , struct st_buffpek *, IO_CACHE *); + my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *, - uint, uint); + uint, ulonglong); } MI_SORT_PARAM; /* functions in mi_check */ diff --git a/include/myisamchk.h b/include/myisamchk.h index c079b7d290b..152467db547 100644 --- a/include/myisamchk.h +++ b/include/myisamchk.h @@ -62,9 +62,10 @@ #define T_ZEROFILL ((ulonglong) 1L << 32) #define T_ZEROFILL_KEEP_LSN ((ulonglong) 1L << 33) /** If repair should not bump create_rename_lsn */ -#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 33) -#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34) -#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35) +#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 34) +#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35) +#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36) +#define T_FORCE_SORT_MEMORY ((ulonglong) 1L << 37) #define T_REP_ANY (T_REP | T_REP_BY_SORT | T_REP_PARALLEL) @@ -180,8 +181,8 @@ typedef struct st_buffpek { my_off_t file_pos; /* Where we are in the sort file */ uchar *base, *key; /* Key pointers */ ha_rows count; /* Number of rows in table */ - ulong mem_count; /* numbers of keys in memory */ - ulong max_keys; /* Max keys in buffert */ + ha_rows mem_count; /* Numbers of keys in memory */ + ha_rows max_keys; /* Max keys in buffert */ } BUFFPEK; #endif /* _myisamchk_h */ diff --git a/mysql-test/r/myisam.result b/mysql-test/r/myisam.result index 2530182ea67..267110be487 100644 --- a/mysql-test/r/myisam.result +++ b/mysql-test/r/myisam.result @@ -2355,7 +2355,7 @@ Warnings: Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '4' REPAIR TABLE t1; Table Op Msg_type Msg_text -test.t1 repair error myisam_sort_buffer_size is too small +test.t1 repair error myisam_sort_buffer_size is too small. X test.t1 repair warning Number of rows changed from 0 to 7168 test.t1 repair status OK SET myisam_repair_threads=2; @@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'); Warnings: -Error 1034 myisam_sort_buffer_size is too small +Error 1034 myisam_sort_buffer_size is too small. X Error 1034 Number of rows changed from 0 to 157 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size; INSERT INTO t1 VALUES('1'); diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index dfd6998b9f3..70b93e7025f 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232 myisam-mmap-size 18446744073709551615 myisam-recover-options DEFAULT myisam-repair-threads 1 -myisam-sort-buffer-size 8388608 +myisam-sort-buffer-size 134216704 myisam-stats-method nulls_unequal myisam-use-mmap FALSE net-buffer-length 16384 diff --git a/mysql-test/r/repair.result b/mysql-test/r/repair.result index 1a98f2f0f36..241c32b4b40 100644 --- a/mysql-test/r/repair.result +++ b/mysql-test/r/repair.result @@ -77,12 +77,12 @@ INSERT INTO t1 VALUES ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'); Warnings: -Error 1034 myisam_sort_buffer_size is too small +Error 1034 myisam_sort_buffer_size is too small. X Error 1034 Number of rows changed from 0 to 157 SET myisam_repair_threads=2; REPAIR TABLE t1; Table Op Msg_type Msg_text -test.t1 repair error myisam_sort_buffer_size is too small +test.t1 repair error myisam_sort_buffer_size is too small. X test.t1 repair warning Number of rows changed from 0 to 157 test.t1 repair status OK SET myisam_repair_threads=@@global.myisam_repair_threads; diff --git a/mysql-test/suite/maria/maria.result b/mysql-test/suite/maria/maria.result index ddd0adade57..914c9568ea4 100644 --- a/mysql-test/suite/maria/maria.result +++ b/mysql-test/suite/maria/maria.result @@ -2679,3 +2679,65 @@ select count(*) from t1; count(*) 13 drop table t1; +# +# BUG#47444 - --myisam_repair_threads > 1 can result in all index +# cardinalities=1 +# +SET aria_repair_threads=2; +SET aria_sort_buffer_size=8192; +CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); +INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); +REPAIR TABLE t1; +Table Op Msg_type Msg_text +test.t1 repair status OK +SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CARDINALITY +14 +14 +14 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +SET aria_sort_buffer_size=@@global.aria_sort_buffer_size; +SET aria_repair_threads=@@global.aria_repair_threads; +# +# BUG#47073 - valgrind errs, corruption,failed repair of partition, +# low myisam_sort_buffer_size +# +CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b)); +INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'), +(6,'0'),(7,'0'); +INSERT INTO t1 SELECT a+10,b FROM t1; +INSERT INTO t1 SELECT a+20,b FROM t1; +INSERT INTO t1 SELECT a+40,b FROM t1; +INSERT INTO t1 SELECT a+80,b FROM t1; +INSERT INTO t1 SELECT a+160,b FROM t1; +INSERT INTO t1 SELECT a+320,b FROM t1; +INSERT INTO t1 SELECT a+640,b FROM t1; +INSERT INTO t1 SELECT a+1280,b FROM t1; +INSERT INTO t1 SELECT a+2560,b FROM t1; +INSERT INTO t1 SELECT a+5120,b FROM t1; +SET aria_sort_buffer_size=4096; +REPAIR TABLE t1; +Table Op Msg_type Msg_text +test.t1 repair error aria_sort_buffer_size is too small. X +test.t1 repair error Create index by sort failed +test.t1 repair info Retrying repair with keycache +test.t1 repair status OK +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +SET aria_repair_threads=2; +REPAIR TABLE t1; +Table Op Msg_type Msg_text +test.t1 repair error aria_sort_buffer_size is too small. X +test.t1 repair error Create index by sort failed +test.t1 repair info Retrying repair with keycache +test.t1 repair status OK +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +SET aria_repair_threads=@@global.aria_repair_threads; +SET aria_sort_buffer_size=@@global.aria_sort_buffer_size; +DROP TABLE t1; diff --git a/mysql-test/suite/maria/maria.test b/mysql-test/suite/maria/maria.test index 3fa7d755fe4..7ecf1e33061 100644 --- a/mysql-test/suite/maria/maria.test +++ b/mysql-test/suite/maria/maria.test @@ -1963,6 +1963,52 @@ unlock tables; select count(*) from t1; drop table t1; +--echo # +--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index +--echo # cardinalities=1 +--echo # +SET aria_repair_threads=2; +SET aria_sort_buffer_size=8192; +CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); +INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); +--replace_regex /Current aria_sort_buffer_size.*/X/ +REPAIR TABLE t1; +SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CHECK TABLE t1; +DROP TABLE t1; +SET aria_sort_buffer_size=@@global.aria_sort_buffer_size; +SET aria_repair_threads=@@global.aria_repair_threads; + +--echo # +--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition, +--echo # low myisam_sort_buffer_size +--echo # +CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b)); +INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'), + (6,'0'),(7,'0'); +INSERT INTO t1 SELECT a+10,b FROM t1; +INSERT INTO t1 SELECT a+20,b FROM t1; +INSERT INTO t1 SELECT a+40,b FROM t1; +INSERT INTO t1 SELECT a+80,b FROM t1; +INSERT INTO t1 SELECT a+160,b FROM t1; +INSERT INTO t1 SELECT a+320,b FROM t1; +INSERT INTO t1 SELECT a+640,b FROM t1; +INSERT INTO t1 SELECT a+1280,b FROM t1; +INSERT INTO t1 SELECT a+2560,b FROM t1; +INSERT INTO t1 SELECT a+5120,b FROM t1; +SET aria_sort_buffer_size=4096; +--replace_regex /Current aria_sort_buffer_size.*/X/ +REPAIR TABLE t1; +CHECK TABLE t1; +SET aria_repair_threads=2; +# May report different values depending on threads activity. +--replace_regex /Current aria_sort_buffer_size.*/X/ +REPAIR TABLE t1; +CHECK TABLE t1; +SET aria_repair_threads=@@global.aria_repair_threads; +SET aria_sort_buffer_size=@@global.aria_sort_buffer_size; +DROP TABLE t1; + # # End of test # diff --git a/mysql-test/suite/maria/maria3.result b/mysql-test/suite/maria/maria3.result index b502d71e772..189329fe5c8 100644 --- a/mysql-test/suite/maria/maria3.result +++ b/mysql-test/suite/maria/maria3.result @@ -317,7 +317,7 @@ aria_pagecache_division_limit 100 aria_page_checksum OFF aria_recover NORMAL aria_repair_threads 1 -aria_sort_buffer_size 134217728 +aria_sort_buffer_size 268434432 aria_stats_method nulls_unequal aria_sync_log_dir NEWFILE show status like 'aria%'; diff --git a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result index cf067c7e7b6..56522566ec9 100644 --- a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result +++ b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result @@ -1,30 +1,34 @@ SET @start_global_value = @@global.aria_sort_buffer_size; select @@global.aria_sort_buffer_size; @@global.aria_sort_buffer_size -134217728 +268434432 select @@session.aria_sort_buffer_size; @@session.aria_sort_buffer_size -134217728 +268434432 show global variables like 'aria_sort_buffer_size'; Variable_name Value -aria_sort_buffer_size 134217728 +aria_sort_buffer_size 268434432 show session variables like 'aria_sort_buffer_size'; Variable_name Value -aria_sort_buffer_size 134217728 +aria_sort_buffer_size 268434432 select * from information_schema.global_variables where variable_name='aria_sort_buffer_size'; VARIABLE_NAME VARIABLE_VALUE -ARIA_SORT_BUFFER_SIZE 134217728 +ARIA_SORT_BUFFER_SIZE 268434432 select * from information_schema.session_variables where variable_name='aria_sort_buffer_size'; VARIABLE_NAME VARIABLE_VALUE -ARIA_SORT_BUFFER_SIZE 134217728 +ARIA_SORT_BUFFER_SIZE 268434432 set global aria_sort_buffer_size=10; +Warnings: +Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10' select @@global.aria_sort_buffer_size; @@global.aria_sort_buffer_size -10 +4096 set session aria_sort_buffer_size=10; +Warnings: +Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10' select @@session.aria_sort_buffer_size; @@session.aria_sort_buffer_size -10 +4096 set global aria_sort_buffer_size=1.1; ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size' set session aria_sort_buffer_size=1e1; @@ -36,7 +40,7 @@ Warnings: Warning 1292 Truncated incorrect aria_sort_buffer_size value: '0' select @@global.aria_sort_buffer_size; @@global.aria_sort_buffer_size -4 +4096 set session aria_sort_buffer_size=cast(-1 as unsigned int); select @@session.aria_sort_buffer_size; @@session.aria_sort_buffer_size diff --git a/mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result b/mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result index 85482be9ad4..932d06668fb 100644 --- a/mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result +++ b/mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result @@ -1,11 +1,11 @@ SET @start_global_value = @@global.myisam_sort_buffer_size ; SELECT @start_global_value; @start_global_value -8388608 +134216704 SET @start_session_value = @@session.myisam_sort_buffer_size ; SELECT @start_session_value; @start_session_value -8388608 +134216704 '#--------------------FN_DYNVARS_005_01-------------------------#' SET @@global.myisam_sort_buffer_size = 100; Warnings: @@ -13,22 +13,22 @@ Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '100' SET @@global.myisam_sort_buffer_size = DEFAULT; SELECT @@global.myisam_sort_buffer_size ; @@global.myisam_sort_buffer_size -8388608 +134216704 SET @@session.myisam_sort_buffer_size = 200; Warnings: Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '200' SET @@session.myisam_sort_buffer_size = DEFAULT; SELECT @@session.myisam_sort_buffer_size ; @@session.myisam_sort_buffer_size -8388608 +134216704 '#--------------------FN_DYNVARS_005_02-------------------------#' SET @@global.myisam_sort_buffer_size = DEFAULT; -SELECT @@global.myisam_sort_buffer_size = 8388608; -@@global.myisam_sort_buffer_size = 8388608 +SELECT @@global.myisam_sort_buffer_size = 134216704; +@@global.myisam_sort_buffer_size = 134216704 1 SET @@session.myisam_sort_buffer_size = DEFAULT; -SELECT @@session.myisam_sort_buffer_size = 8388608; -@@session.myisam_sort_buffer_size = 8388608 +SELECT @@session.myisam_sort_buffer_size = 134216704; +@@session.myisam_sort_buffer_size = 134216704 1 '#--------------------FN_DYNVARS_005_03-------------------------#' SET @@global.myisam_sort_buffer_size = 4; @@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list' SET @@global.myisam_sort_buffer_size = @start_global_value; SELECT @@global.myisam_sort_buffer_size ; @@global.myisam_sort_buffer_size -8388608 +134216704 SET @@session.myisam_sort_buffer_size = @start_session_value; SELECT @@session.myisam_sort_buffer_size ; @@session.myisam_sort_buffer_size -8388608 +134216704 diff --git a/mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test b/mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test index c35949ac5f3..1aa25fa2c8e 100644 --- a/mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test +++ b/mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test @@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size ; ######################################################################## SET @@global.myisam_sort_buffer_size = DEFAULT; -SELECT @@global.myisam_sort_buffer_size = 8388608; +SELECT @@global.myisam_sort_buffer_size = 134216704; SET @@session.myisam_sort_buffer_size = DEFAULT; -SELECT @@session.myisam_sort_buffer_size = 8388608; +SELECT @@session.myisam_sort_buffer_size = 134216704; --echo '#--------------------FN_DYNVARS_005_03-------------------------#' diff --git a/mysql-test/t/myisam.test b/mysql-test/t/myisam.test index 8323890b028..6c9371eed25 100644 --- a/mysql-test/t/myisam.test +++ b/mysql-test/t/myisam.test @@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1; INSERT INTO t1 SELECT a+2560,b FROM t1; INSERT INTO t1 SELECT a+5120,b FROM t1; SET myisam_sort_buffer_size=4; +--replace_regex /Current myisam_sort_buffer_size.*/X/ REPAIR TABLE t1; SET myisam_repair_threads=2; @@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3; CREATE TABLE t1(a CHAR(255), KEY(a)); SELECT * FROM t1, t1 AS a1; SET myisam_sort_buffer_size=4; +--replace_regex /Current myisam_sort_buffer_size.*/X/ INSERT INTO t1 VALUES ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), diff --git a/mysql-test/t/repair.test b/mysql-test/t/repair.test index 3c55f06ff4c..6536c052019 100644 --- a/mysql-test/t/repair.test +++ b/mysql-test/t/repair.test @@ -61,6 +61,7 @@ DROP TABLE t1; # CREATE TABLE t1(a CHAR(255), KEY(a)); SET myisam_sort_buffer_size=4096; +--replace_regex /Current myisam_sort_buffer_size.*/X/ INSERT INTO t1 VALUES ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), @@ -79,6 +80,7 @@ INSERT INTO t1 VALUES ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'); SET myisam_repair_threads=2; +--replace_regex /Current myisam_sort_buffer_size.*/X/ REPAIR TABLE t1; SET myisam_repair_threads=@@global.myisam_repair_threads; SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 1a11bdd6050..6561bec5f22 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, "disables parallel repair.", 0, 0, 1, 1, 128, 1); -static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, +static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, "The buffer that is allocated when sorting the index when doing a " - "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", - 0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1); + "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL, + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1); static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, "Specifies how Aria index statistics collection code should treat " @@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) if ((param.testflag & T_REP_BY_SORT)) { param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP; + if (thd->vio_ok()) + _ma_check_print_info(¶m, "Retrying repair with keycache"); sql_print_information("Retrying repair of: '%s' with keycache", table->s->path.str); continue; diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index 8d2a2d3aec3..6bcb9c21a71 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param) param->keys_in_use= ~(ulonglong) 0; param->search_after_block=HA_OFFSET_ERROR; param->auto_increment_value= 0; - param->use_buffers=USE_BUFFER_INIT; + param->use_buffers= PAGE_BUFFER_INIT; param->read_buffer_length=READ_BUFFER_INIT; param->write_buffer_length=READ_BUFFER_INIT; param->sort_buffer_length=SORT_BUFFER_INIT; diff --git a/storage/maria/ma_sort.c b/storage/maria/ma_sort.c index 4bc179c3008..65035e77c76 100644 --- a/storage/maria/ma_sort.c +++ b/storage/maria/ma_sort.c @@ -36,8 +36,10 @@ #define MERGEBUFF2 31 #define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD) #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) -#define DISK_BUFFER_SIZE (IO_SIZE*16) +#define DISK_BUFFER_SIZE (IO_SIZE*128) +/* How many keys we can keep in memory */ +typedef ulonglong ha_keys; /* Pointers of functions for store and read keys from temp file @@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...); /* Functions defined in this file */ -static ha_rows find_all_keys(MARIA_SORT_PARAM *info,uint keys, +static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, - DYNAMIC_ARRAY *buffpek,int *maxbuffer, + DYNAMIC_ARRAY *buffpek,uint *maxbuffer, IO_CACHE *tempfile, IO_CACHE *tempfile_for_exceptions); -static int write_keys(MARIA_SORT_PARAM *info, uchar **sort_keys, - uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); +static int write_keys(MARIA_SORT_PARAM *info,uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile); static int write_key(MARIA_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile); static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys, - uint count); -static int merge_many_buff(MARIA_SORT_PARAM *info,uint keys, + ha_keys count); +static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, - BUFFPEK *buffpek,int *maxbuffer, + BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file); -static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, - uint sort_length); -static int merge_buffers(MARIA_SORT_PARAM *info,uint keys, +static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb); -static int merge_index(MARIA_SORT_PARAM *,uint, uchar **,BUFFPEK *, int, +static int merge_index(MARIA_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint, IO_CACHE *); static int flush_maria_ft_buf(MARIA_SORT_PARAM *info); -static int write_keys_varlen(MARIA_SORT_PARAM *info, uchar **sort_keys, - uint count, BUFFPEK *buffpek, +static int write_keys_varlen(MARIA_SORT_PARAM *info,uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile); -static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, - uint sort_length); +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file, - uchar *key, uint sort_length, uint count); + uchar *key, uint sort_length, ha_keys count); static int write_merge_key_varlen(MARIA_SORT_PARAM *info, - IO_CACHE *to_file, uchar *key, - uint sort_length, uint count); + IO_CACHE *to_file, + uchar* key, uint sort_length, + ha_keys count); static inline int my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); @@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, size_t sortbuff_size) { - int error,maxbuffer,skr; - size_t memavl,old_memavl; - uint keys,sort_length; + int error; + uint sort_length, maxbuffer; + size_t memavl, old_memavl; DYNAMIC_ARRAY buffpek; - ha_rows records; + ha_rows records, keys; uchar **sort_keys; IO_CACHE tempfile, tempfile_for_exceptions; DBUG_ENTER("_ma_create_index_by_sort"); @@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, if (info->keyinfo->flag & HA_VAR_LENGTH_KEY) { info->write_keys= write_keys_varlen; - info->read_to_buffer=read_to_buffer_varlen; + info->read_to_buffer= read_to_buffer_varlen; info->write_key=write_merge_key_varlen; } else @@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, while (memavl >= MIN_SORT_MEMORY) { - if ((records < UINT_MAX32) && - ((my_off_t) (records + 1) * - (sort_length + sizeof(char*)) <= (my_off_t) memavl)) - keys= (uint)records+1; + /* Check if we can fit all keys into memory */ + if (((ulonglong) (records + 1) * + (sort_length + sizeof(char*)) <= memavl)) + keys= records+1; + else if ((info->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1); + } else + { + /* + All keys can't fit in memory. + Calculate how many keys + buffers we can keep in memory + */ + uint maxbuffer_org; do { - skr=maxbuffer; - if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer || - (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/ + maxbuffer_org= maxbuffer; + if (memavl < sizeof(BUFFPEK) * maxbuffer || + (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/ (sort_length+sizeof(char*))) <= 1 || - keys < (uint) maxbuffer) + keys < maxbuffer) { _ma_check_print_error(info->sort_info->param, - "aria_sort_buffer_size is too small"); + "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u", + (ulonglong) sortbuff_size, (ulonglong) records, + sort_length); + my_errno= ENOMEM; goto err; } } - while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); + while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org); + } if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+ HA_FT_MAXBYTELEN, MYF(0)))) { if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer, - maxbuffer/2, MYF(0))) + min(maxbuffer/2, 1000), MYF(0))) { my_free(sort_keys); sort_keys= 0; @@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, } if (memavl < MIN_SORT_MEMORY) { - _ma_check_print_error(info->sort_info->param, "Aria sort buffer" - " too small"); /* purecov: tested */ - goto err; /* purecov: tested */ + /* purecov: begin inspected */ + _ma_check_print_error(info->sort_info->param, + "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u", + (ulonglong) sortbuff_size, (ulonglong) records, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ } (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ if (!no_messages) - printf(" - Searching for keys, allocating buffer for %d keys\n",keys); + my_fprintf(stdout, + " - Searching for keys, allocating buffer for %llu keys\n", + (ulonglong) keys); if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, &tempfile,&tempfile_for_exceptions)) @@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, if (maxbuffer == 0) { if (!no_messages) - printf(" - Dumping %lu keys\n", (ulong) records); - if (write_index(info,sort_keys, (uint) records)) + my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records); + if (write_index(info, sort_keys, (ha_keys) records)) goto err; /* purecov: inspected */ } else @@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, if (maxbuffer >= MERGEBUFF2) { if (!no_messages) - printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */ + my_fprintf(stdout, " - Merging %llu keys\n", + (ulonglong) records); /* purecov: tested */ if (merge_many_buff(info,keys,sort_keys, dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) goto err; /* purecov: inspected */ @@ -266,13 +298,13 @@ err: /* Search after all keys and place them in a temp. file */ -static ha_rows find_all_keys(MARIA_SORT_PARAM *info, uint keys, +static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_rows keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek, - int *maxbuffer, IO_CACHE *tempfile, + uint *maxbuffer, IO_CACHE *tempfile, IO_CACHE *tempfile_for_exceptions) { int error; - uint idx; + ha_rows idx; DBUG_ENTER("find_all_keys"); idx=error=0; @@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) { MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg; int error; - size_t memavl,old_memavl; + size_t memavl, old_memavl; + longlong sortbuff_size; + ha_keys keys, idx; uint sort_length; - ulong idx, maxbuffer, keys; + uint maxbuffer; uchar **sort_keys=0; LINT_INIT(keys); @@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek)); bzero((char*) &sort_param->unique, sizeof(sort_param->unique)); - memavl= max(sort_param->sortbuff_size, MIN_SORT_MEMORY); - idx= (uint)sort_param->sort_info->max_records; + sortbuff_size= sort_param->sortbuff_size; + memavl= max(sortbuff_size, MIN_SORT_MEMORY); + idx= (ha_keys) sort_param->sort_info->max_records; sort_length= sort_param->key_length; maxbuffer= 1; @@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) { if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl) keys= idx+1; + else if ((sort_param->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1); + } else { - ulong skr; + uint maxbuffer_org; do { - skr= maxbuffer; + maxbuffer_org= maxbuffer; if (memavl < sizeof(BUFFPEK)*maxbuffer || (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ (sort_length+sizeof(char*))) <= 1 || keys < maxbuffer) { _ma_check_print_error(sort_param->sort_info->param, - "aria_sort_buffer_size is too small"); + "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); goto err; } } - while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr); + while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org); } if ((sort_keys= (uchar **) my_malloc(keys*(sort_length+sizeof(char*))+ @@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) HA_FT_MAXBYTELEN : 0), MYF(0)))) { if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK), - maxbuffer, maxbuffer/2, MYF(0))) + maxbuffer, min(maxbuffer/2, 1000), MYF(0))) { my_free(sort_keys); sort_keys= (uchar **) NULL; /* for err: label */ @@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) } if (memavl < MIN_SORT_MEMORY) { + /* purecov: begin inspected */ _ma_check_print_error(sort_param->sort_info->param, - "Aria sort buffer too small"); - goto err; /* purecov: tested */ + "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ } if (sort_param->sort_info->param->testflag & T_VERBOSE) - printf("Key %d - Allocating buffer for %lu keys\n", - sort_param->key+1, (ulong) keys); + my_fprintf(stdout, + "Key %d - Allocating buffer for %llu keys\n", + sort_param->key + 1, (ulonglong) keys); sort_param->sort_keys= sort_keys; idx= error= 0; @@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) { if (sort_param->real_key_length > sort_param->key_length) { - if (write_key(sort_param,sort_keys[idx], + if (write_key(sort_param, sort_keys[idx], &sort_param->tempfile_for_exceptions)) goto err; continue; @@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) (size_t) sort_param->key_length); idx= 1; } - sort_keys[idx]=sort_keys[idx - 1] + sort_param->key_length; + sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length; } if (error > 0) goto err; @@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) else sort_param->keys= idx; - sort_param->sort_keys_length= keys; goto ok; err: DBUG_PRINT("error", ("got some error")); sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */ my_free(sort_keys); - sort_param->sort_keys=0; + sort_param->sort_keys= 0; delete_dynamic(& sort_param->buffpek); close_cached_file(&sort_param->tempfile); close_cached_file(&sort_param->tempfile_for_exceptions); @@ -483,8 +535,12 @@ ok: Detach from the share if the writer is involved. Avoid others to be blocked. This includes a flush of the write buffer. This will also indicate EOF to the readers. + That means that a writer always gets here first and readers - + only when they see EOF. But if a reader finishes prematurely + because of an error it may reach this earlier - don't allow it + to detach the writer thread. */ - if (sort_param->sort_info->info->rec_cache.share) + if (sort_param->master && sort_param->sort_info->info->rec_cache.share) remove_io_thread(&sort_param->sort_info->info->rec_cache); /* Readers detach from the share if any. Avoid others to be blocked. */ @@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) for (i= 0, sinfo= sort_param ; i < sort_info->total_keys ; - i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++) + i++, sinfo++) { if (!sinfo->sort_keys) { @@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) { if (param->testflag & T_VERBOSE) { - printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys); + my_fprintf(stdout, + "Key %d - Dumping %llu keys\n", sinfo->key+1, + (ulonglong) sinfo->keys); fflush(stdout); } if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo)) got_error=1; } - if (!got_error && param->testflag & T_STATISTICS) - maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique, - param->stats_method == - MI_STATS_METHOD_IGNORE_NULLS ? - sinfo->notnull : NULL, - (ulonglong) share->state.state.records); } my_free(sinfo->sort_keys); my_free(sinfo->rec_buff); @@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) delete_dynamic(&sinfo->buffpek), close_cached_file(&sinfo->tempfile), close_cached_file(&sinfo->tempfile_for_exceptions), + rec_per_key_part+= sinfo->keyinfo->keysegs, sinfo++) { if (got_error) @@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) if (maxbuffer >= MERGEBUFF2) { if (param->testflag & T_VERBOSE) - printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); - if (merge_many_buff(sinfo, keys, (uchar **) mergebuf, + my_fprintf(stdout, + "Key %d - Merging %llu keys\n", + sinfo->key+1, (ulonglong) sinfo->keys); + if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), - (int*) &maxbuffer, &sinfo->tempfile)) + &maxbuffer, &sinfo->tempfile)) { got_error=1; continue; @@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) } } } + if (!got_error && (param->testflag & T_STATISTICS)) + maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique, + param->stats_method == + MI_STATS_METHOD_IGNORE_NULLS ? + sinfo->notnull : NULL, + (ulonglong) share->state.state.records); + } my_free(mergebuf); DBUG_RETURN(got_error); @@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) /* Write all keys in memory to file for later merge */ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys, - uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile) { uchar **end; uint sort_length=info->key_length; DBUG_ENTER("write_keys"); + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, info); if (!my_b_inited(tempfile) && @@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys, for (end=sort_keys+count ; sort_keys != end ; sort_keys++) { - if (my_b_write(tempfile, *sort_keys, (uint) sort_length)) + if (my_b_write(tempfile, *sort_keys, sort_length)) DBUG_RETURN(1); /* purecov: inspected */ } DBUG_RETURN(0); @@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs) static int write_keys_varlen(MARIA_SORT_PARAM *info, - register uchar **sort_keys, - uint count, BUFFPEK *buffpek, - IO_CACHE *tempfile) + register uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek, + IO_CACHE *tempfile) { uchar **end; int err; DBUG_ENTER("write_keys_varlen"); + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, info); if (!my_b_inited(tempfile) && @@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key, /* Write index */ -static int write_index(MARIA_SORT_PARAM *info, - register uchar **sort_keys, - register uint count) +static int write_index(MARIA_SORT_PARAM *info, register uchar **sort_keys, + register ha_keys count) { DBUG_ENTER("write_index"); @@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info, /* Merge buffers to make < MERGEBUFF2 buffers */ -static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, - uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *t_file) +static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys, + uchar **sort_keys, BUFFPEK *buffpek, + uint *maxbuffer, IO_CACHE *t_file) { - int tmp, merges, max_merges; + uint tmp, merges, max_merges; IO_CACHE t_file2, *from_file, *to_file, *temp; BUFFPEK *lastbuff; DBUG_ENTER("merge_many_buff"); @@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, from_file= t_file ; to_file= &t_file2; while (*maxbuffer >= MERGEBUFF2) { - int i; + uint i; reinit_io_cache(from_file,READ_CACHE,0L,0,0); reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); lastbuff=buffpek; - for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) + for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF) { if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, buffpek+i,buffpek+i+MERGEBUFF-1)) @@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, if (flush_io_cache(to_file)) break; /* purecov: inspected */ temp=from_file; from_file=to_file; to_file=temp; - *maxbuffer= (int) (lastbuff-buffpek)-1; + *maxbuffer= (uint) (lastbuff-buffpek)-1; if (info->sort_info->param->max_stage != 1) /* If not parallel */ _ma_report_progress(info->sort_info->param, merges++, max_merges); } cleanup: close_cached_file(to_file); /* This holds old result */ if (to_file == t_file) + { + DBUG_ASSERT(t_file2.type == WRITE_CACHE); *t_file=t_file2; /* Copy result file */ + t_file->current_pos= &t_file->write_pos; + t_file->current_end= &t_file->write_end; + } DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ } /* merge_many_buff */ @@ -851,33 +923,35 @@ cleanup: -1 Error */ -static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, - uint sort_length) +static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) { - register uint count; - uint length; + register ha_keys count; + my_off_t length; - if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count))) { - if (mysql_file_pread(fromfile->file, buffpek->base, - (length= sort_length*count),buffpek->file_pos,MYF_RW)) - return((uint) -1); /* purecov: inspected */ + if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base, + (length= sort_length * count), + buffpek->file_pos, MYF_RW)) + return(HA_OFFSET_ERROR); /* purecov: inspected */ buffpek->key=buffpek->base; buffpek->file_pos+= length; /* New filepos */ buffpek->count-= count; buffpek->mem_count= count; } - return (count*sort_length); + return (((my_off_t) count) * sort_length); } /* read_to_buffer */ -static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, - uint sort_length) + +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) { - register uint count; + register ha_keys count; uint idx; uchar *buffp; - if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count))) { buffp= buffpek->base; @@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, uint16 length_of_key; if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key), buffpek->file_pos,MYF_RW)) - return((uint) -1); + return(HA_OFFSET_ERROR); buffpek->file_pos+=sizeof(length_of_key); if (mysql_file_pread(fromfile->file, buffp, length_of_key, buffpek->file_pos,MYF_RW)) @@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, buffpek->count-= count; buffpek->mem_count= count; } - return (count*sort_length); + return (((my_off_t) count) * sort_length); } /* read_to_buffer_varlen */ static int write_merge_key_varlen(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar* key, - uint sort_length, uint count) + uint sort_length, ha_keys count) { - uint idx; + ha_keys idx; uchar *bufs = key; for (idx=1;idx<=count;idx++) @@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info, static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)), - IO_CACHE *to_file, uchar *key, - uint sort_length, uint count) + IO_CACHE *to_file, uchar *key, + uint sort_length, ha_keys count) { - return my_b_write(to_file, key, (size_t) sort_length*count); + return my_b_write(to_file, key, ((size_t) sort_length) * count); } /* Merge buffers to one buffer If to_file == 0 then use info->key_write + + Return: + 0 ok + 1 error */ static int -merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, +merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb) { - int error; - uint sort_length,maxcount; + int error= 1; + uint sort_length; + ha_keys maxcount; ha_rows count; - my_off_t UNINIT_VAR(to_start_filepos); + my_off_t UNINIT_VAR(to_start_filepos), read_length; uchar *strpos; BUFFPEK *buffpek,**refpek; QUEUE queue; DBUG_ENTER("merge_buffers"); - count=error=0; - maxcount=keys/((uint) (Tb-Fb) +1); + count= 0; + maxcount= keys/((uint) (Tb-Fb) +1); DBUG_ASSERT(maxcount > 0); + LINT_INIT(to_start_filepos); if (to_file) to_start_filepos=my_b_tell(to_file); strpos= (uchar*) sort_keys; @@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { count+= buffpek->count; buffpek->base= strpos; - buffpek->max_keys=maxcount; - strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek, - sort_length)); - if (error == -1) + buffpek->max_keys= maxcount; + strpos+= (read_length= info->read_to_buffer(from_file,buffpek, + sort_length)); + if (read_length == HA_OFFSET_ERROR) goto err; /* purecov: inspected */ queue_insert(&queue,(uchar*) buffpek); } @@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { if (info->write_key(info,to_file, buffpek->key, (uint) sort_length,1)) - { - error=1; goto err; /* purecov: inspected */ - } + goto err; /* purecov: inspected */ } else { if ((*info->key_write)(info,(void*) buffpek->key)) - { - error=1; goto err; /* purecov: inspected */ - } + goto err; /* purecov: inspected */ } buffpek->key+=sort_length; if (! --buffpek->mem_count) { /* It's enough to check for killedptr before a slow operation */ if (_ma_killed_ptr(info->sort_info->param)) - { - error=1; goto err; - } - if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length))) + if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length))) { uchar *base= buffpek->base; uint max_keys=buffpek->max_keys; @@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } break; /* One buffer have been removed */ } + else if (read_length == HA_OFFSET_ERROR) + goto err; /* purecov: inspected */ } - else if (error == -1) - goto err; /* purecov: inspected */ queue_replace_top(&queue); /* Top element has been replaced */ } } @@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } } } - while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != - -1 && error != 0); + while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0); + if (read_length == 0) + error= 0; lastbuff->count=count; if (to_file) @@ -1076,8 +1150,8 @@ err: /* Do a merge to output-file (save only positions) */ static int -merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys, - BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) +merge_index(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, + BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile) { DBUG_ENTER("merge_index"); if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index 62225a7a4b0..8a7ba30eb08 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -191,7 +191,7 @@ end: enum options_mc { OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS, - OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE, + OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE, OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE, OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, @@ -229,6 +229,11 @@ static struct my_option my_long_options[] = {"correct-checksum", OPT_CORRECT_CHECKSUM, "Correct checksum information for table.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create-missing-keys", OPT_CREATE_MISSING_KEYS, + "Create missing keys. This assumes that the data file is correct and that " + "the the number of rows stored in the index file is correct. Enables " + "--quick", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.", @@ -362,8 +367,8 @@ static struct my_option my_long_options[] = { "page_buffer_size", OPT_PAGE_BUFFER_SIZE, "Size of page buffer. Used by --safe-repair", &check_param.use_buffers, &check_param.use_buffers, 0, - GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L, - (long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0}, + GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L, + SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0}, { "read_buffer_size", OPT_READ_BUFFER_SIZE, "Read buffer size for sequential reads during scanning", &check_param.read_buffer_length, @@ -379,9 +384,8 @@ static struct my_option my_long_options[] = { "sort_buffer_size", OPT_SORT_BUFFER_SIZE, "Size of sort buffer. Used by --recover", &check_param.sort_buffer_length, - &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG, - (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD), - (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0}, + &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG, + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0}, { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "Internal buffer for sorting keys; Don't touch :)", &check_param.sort_key_blocks, @@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\ --correct-checksum Correct checksum information for table.\n\ -D, --data-file-length=# Max length of data file (when recreating data\n\ file when it's full).\n\ + --create-missing-keys\n\ + Create missing keys. This assumes that the data\n\ + file is correct and that the the number of rows stored\n\ + in the index file is correct. Enables --quick.\n\ -e, --extend-check Try to recover every possible row from the data file\n\ Normally this will also find a lot of garbage rows;\n\ Don't use this option if you are not totally desperate.\n\ - -f, --force Overwrite old temporary files.\n\ + -f, --force Overwrite old temporary files. Add another --force to\n\ + avoid 'sort_buffer_size is too small' errors.\n\ + In this case we will attempt to do the repair with the\n\ + given sort_buffer_size and dynamically allocate\n\ + as many management buffers as needed.\n\ -k, --keys-used=# Tell Aria to update only some specific keys. # is a\n\ bit mask of which keys to use. This can be used to\n\ get faster inserts.\n\ @@ -664,10 +676,13 @@ get_one_option(int optid, if (argument == disabled_my_option) { check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; - check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE); + check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE | + T_FORCE_SORT_MEMORY); } else { + if (check_param.testflag & T_FORCE_CREATE) + check_param.testflag= T_FORCE_SORT_MEMORY; check_param.tmpfile_createflag= O_RDWR | O_TRUNC; check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; } @@ -720,8 +735,26 @@ get_one_option(int optid, if (argument == disabled_my_option) check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); else + { + /* + If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS, + then add T_FORCE_UNIQUENESS. + */ check_param.testflag|= - (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK; + ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) == + T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK); + } + break; + case OPT_CREATE_MISSING_KEYS: + if (argument == disabled_my_option) + check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS); + else + { + check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS; + /* Use repair by sort by default */ + if (!(check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_REP_BY_SORT; + } break; case 'u': if (argument == disabled_my_option) diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index 0dce9114f2a..a1275e7ccd2 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -94,9 +94,10 @@ typedef struct st_maria_sort_param */ ulonglong unique[HA_MAX_KEY_SEG+1]; ulonglong notnull[HA_MAX_KEY_SEG+1]; + ulonglong sortbuff_size; MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos; - uint key, key_length,real_key_length,sortbuff_size; + uint key, key_length,real_key_length; uint maxbuffers, keys, find_length, sort_keys_length; my_bool fix_datafile, master; my_bool calc_checksum; /* calculate table checksum */ @@ -107,10 +108,10 @@ typedef struct st_maria_sort_param int (*key_write)(struct st_maria_sort_param *, const uchar *); void (*lock_in_memory)(HA_CHECK *); int (*write_keys)(struct st_maria_sort_param *, register uchar **, - uint , struct st_buffpek *, IO_CACHE *); - uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); + ulonglong , struct st_buffpek *, IO_CACHE *); + my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *, - uint, uint); + uint, ulonglong); } MARIA_SORT_PARAM; int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile); @@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info #define UPDATE_AUTO_INC 8 #define UPDATE_OPEN_COUNT 16 -#define USE_BUFFER_INIT (((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192) -#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD) -#define SORT_BUFFER_INIT (1024L*1024L*256-MALLOC_OVERHEAD) -#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD) +/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */ +#define PAGE_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192) +#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024) +#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024) +#define MIN_SORT_BUFFER 4096 #define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0) #define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1) diff --git a/storage/maria/maria_ftdump.c b/storage/maria/maria_ftdump.c index 92e57fa9a1e..68e13a8ddc4 100644 --- a/storage/maria/maria_ftdump.c +++ b/storage/maria/maria_ftdump.c @@ -84,7 +84,7 @@ int main(int argc,char *argv[]) usage(); } - init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0, + init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0, MARIA_KEY_BLOCK_LENGTH, MY_WME); if (!(info=maria_open(argv[0], O_RDONLY, diff --git a/storage/maria/maria_read_log.c b/storage/maria/maria_read_log.c index 74aa8bd9d11..d9393399da4 100644 --- a/storage/maria/maria_read_log.c +++ b/storage/maria/maria_read_log.c @@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace"; static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent; static my_bool opt_check; static const char *opt_tmpdir; -static ulong opt_page_buffer_size, opt_translog_buffer_size; +static ulong opt_translog_buffer_size; +static ulonglong opt_page_buffer_size; static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint; static MY_TMPDIR maria_chk_tmpdir; @@ -204,9 +205,8 @@ static struct my_option my_long_options[] = { "page-buffer-size", 'P', "The size of the buffer used for index blocks for Aria tables", &opt_page_buffer_size, &opt_page_buffer_size, 0, - GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, - 1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD, - (long) IO_SIZE, 0}, + GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT, + PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0}, { "start-from-lsn", 'o', "Start reading log from this lsn", &opt_start_from_lsn, &opt_start_from_lsn, 0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 }, diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 5432f4fa605..c5d4f987c00 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, "The buffer that is allocated when sorting the index when doing " "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL, - 8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1); + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1); static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG, "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE); diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index 056aff5a72b..97ea1d17c26 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param) param->opt_follow_links=1; param->keys_in_use= ~(ulonglong) 0; param->search_after_block=HA_OFFSET_ERROR; - param->use_buffers=USE_BUFFER_INIT; + param->use_buffers= KEY_BUFFER_INIT; param->read_buffer_length=READ_BUFFER_INIT; param->write_buffer_length=READ_BUFFER_INIT; param->sort_buffer_length=SORT_BUFFER_INIT; diff --git a/storage/myisam/myisam_ftdump.c b/storage/myisam/myisam_ftdump.c index ce0d7be3f0b..e1ea9f2de37 100644 --- a/storage/myisam/myisam_ftdump.c +++ b/storage/myisam/myisam_ftdump.c @@ -84,7 +84,7 @@ int main(int argc,char *argv[]) usage(); } - init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0); + init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0); if (!(info=mi_open(argv[0], O_RDONLY, HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER))) diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c index 7be84599d86..d5c13e9cba3 100644 --- a/storage/myisam/myisamchk.c +++ b/storage/myisam/myisamchk.c @@ -134,7 +134,7 @@ int main(int argc, char **argv) enum options_mc { OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS, - OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE, + OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE, OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE, OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, @@ -165,6 +165,11 @@ static struct my_option my_long_options[] = {"correct-checksum", OPT_CORRECT_CHECKSUM, "Correct checksum information for table.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create-missing-keys", OPT_CREATE_MISSING_KEYS, + "Create missing keys. This assumes that the data file is correct and that " + "the the number of rows stored in the index file is correct. Enables " + "--quick", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.", @@ -270,7 +275,7 @@ static struct my_option my_long_options[] = 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "", &check_param.use_buffers, &check_param.use_buffers, 0, - GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD, + GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD, SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0}, { "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "", &opt_key_cache_block_size, @@ -395,10 +400,18 @@ static void usage(void) -e, --extend-check Try to recover every possible row from the data file\n\ Normally this will also find a lot of garbage rows;\n\ Don't use this option if you are not totally desperate.\n\ - -f, --force Overwrite old temporary files.\n\ + -f, --force Overwrite old temporary files. Add another --force to\n\ + avoid 'myisam_sort_buffer_size is too small' errors.\n\ + In this case we will attempt to do the repair with the\n\ + given myisam_sort_buffer_size and dynamically allocate\n\ + as many management buffers as needed.\n\ -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ bit mask of which keys to use. This can be used to\n\ get faster inserts.\n\ + --create-missing-keys\n\ + Create missing keys. This assumes that the data\n\ + file is correct and that the the number of rows stored\n\ + in the index file is correct. Enables --quick\n\ --max-record-length=#\n\ Skip rows bigger than this if myisamchk can't allocate\n\ memory to hold it.\n\ @@ -541,10 +554,13 @@ get_one_option(int optid, if (argument == disabled_my_option) { check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; - check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE); + check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE | + T_FORCE_SORT_MEMORY); } else { + if (check_param.testflag & T_FORCE_CREATE) + check_param.testflag= T_FORCE_SORT_MEMORY; check_param.tmpfile_createflag= O_RDWR | O_TRUNC; check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; } @@ -597,8 +613,26 @@ get_one_option(int optid, if (argument == disabled_my_option) check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); else + { + /* + If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS, + then add T_FORCE_UNIQUENESS. + */ check_param.testflag|= - (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK; + ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) == + T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK); + } + break; + case OPT_CREATE_MISSING_KEYS: + if (argument == disabled_my_option) + check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS); + else + { + check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS; + /* Use repair by sort by default */ + if (!(check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_REP_BY_SORT; + } break; case 'u': if (argument == disabled_my_option) diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h index a0377e34130..178660d6249 100644 --- a/storage/myisam/myisamdef.h +++ b/storage/myisam/myisamdef.h @@ -629,10 +629,11 @@ typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */ #define UPDATE_AUTO_INC 8 #define UPDATE_OPEN_COUNT 16 -#define USE_BUFFER_INIT (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE) -#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD) -#define SORT_BUFFER_INIT (2048L*1024L-MALLOC_OVERHEAD) -#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD) +/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */ +#define KEY_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE) +#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024) +#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024) +#define MIN_SORT_BUFFER 4096 enum myisam_log_commands { diff --git a/storage/myisam/sort.c b/storage/myisam/sort.c index 4af45ea02e9..4bff7b14e1a 100644 --- a/storage/myisam/sort.c +++ b/storage/myisam/sort.c @@ -34,8 +34,10 @@ #define MERGEBUFF 15 #define MERGEBUFF2 31 #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) -#define DISK_BUFFER_SIZE (IO_SIZE*16) +#define DISK_BUFFER_SIZE (IO_SIZE*128) +/* How many keys we can keep in memory */ +typedef ulonglong ha_keys; /* Pointers of functions for store and read keys from temp file @@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...); /* Functions defined in this file */ -static ha_rows find_all_keys(MI_SORT_PARAM *info,uint keys, +static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, - DYNAMIC_ARRAY *buffpek,int *maxbuffer, + DYNAMIC_ARRAY *buffpek,uint *maxbuffer, IO_CACHE *tempfile, IO_CACHE *tempfile_for_exceptions); static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys, - uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); + ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile); static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile); static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys, - uint count); -static int merge_many_buff(MI_SORT_PARAM *info,uint keys, + ha_keys count); +static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys, uchar * *sort_keys, - BUFFPEK *buffpek,int *maxbuffer, + BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file); -static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, - uint sort_length); -static int merge_buffers(MI_SORT_PARAM *info,uint keys, +static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar * *sort_keys, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb); -static int merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, +static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint, IO_CACHE *); static int flush_ft_buf(MI_SORT_PARAM *info); static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, - uint count, BUFFPEK *buffpek, + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile); -static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, uint sort_length); static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file, - uchar *key, uint sort_length, uint count); + uchar *key, uint sort_length, ha_keys count); static int write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar* key, uint sort_length, - uint count); + ha_keys count); static inline int my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); @@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulonglong sortbuff_size) { - int error,maxbuffer,skr; - uint sort_length, keys; + int error; + uint sort_length, maxbuffer; ulonglong memavl, old_memavl; DYNAMIC_ARRAY buffpek; - ha_rows records; + ha_rows records, keys; uchar **sort_keys; IO_CACHE tempfile, tempfile_for_exceptions; DBUG_ENTER("_create_index_by_sort"); - DBUG_PRINT("enter",("sort_length: %d", info->key_length)); + DBUG_PRINT("enter",("sort_length: %u", info->key_length)); if (info->keyinfo->flag & HA_VAR_LENGTH_KEY) { @@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, sort_length= info->key_length; LINT_INIT(keys); - if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32) - memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *)); - while (memavl >= MIN_SORT_BUFFER) { - if ((records < UINT_MAX32) && - ((my_off_t) (records + 1) * - (sort_length + sizeof(char*)) <= (my_off_t) memavl)) - keys= (uint)records+1; + /* Check if we can fit all keys into memory */ + if (((ulonglong) (records + 1) * + (sort_length + sizeof(char*)) <= memavl)) + keys= records+1; + else if ((info->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1); + } else + { + /* + All keys can't fit in memory. + Calculate how many keys + buffers we can keep in memory + */ + uint maxbuffer_org; do { - skr=maxbuffer; - if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer || - (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/ + maxbuffer_org= maxbuffer; + if (memavl < sizeof(BUFFPEK) * maxbuffer || + (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/ (sort_length+sizeof(char*))) <= 1 || - keys < (uint) maxbuffer) + keys < maxbuffer) { mi_check_print_error(info->sort_info->param, - "myisam_sort_buffer_size is too small"); + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) records, + sort_length); my_errno= ENOMEM; goto err; } } - while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); + while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org); + } if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+ HA_FT_MAXBYTELEN, MYF(0)))) { if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer, - maxbuffer/2, MYF(0))) + min(maxbuffer/2, 1000), MYF(0))) { my_free(sort_keys); sort_keys= 0; @@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, } if (memavl < MIN_SORT_BUFFER) { - mi_check_print_error(info->sort_info->param,"MyISAM sort buffer too small"); /* purecov: tested */ - my_errno= ENOMEM; /* purecov: tested */ - goto err; /* purecov: tested */ + /* purecov: begin inspected */ + mi_check_print_error(info->sort_info->param, + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) records, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ } (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ if (!no_messages) - printf(" - Searching for keys, allocating buffer for %d keys\n",keys); + my_fprintf(stdout, + " - Searching for keys, allocating buffer for %llu keys\n", + (ulonglong) keys); - if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, - &tempfile,&tempfile_for_exceptions)) + if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, + &tempfile,&tempfile_for_exceptions)) == HA_POS_ERROR) goto err; /* purecov: tested */ if (maxbuffer == 0) { if (!no_messages) - printf(" - Dumping %lu keys\n", (ulong) records); - if (write_index(info,sort_keys, (uint) records)) + my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records); + if (write_index(info, sort_keys, (ha_keys) records)) goto err; /* purecov: inspected */ } else @@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (maxbuffer >= MERGEBUFF2) { if (!no_messages) - printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */ + my_fprintf(stdout, " - Merging %llu keys\n", + (ulonglong) records); /* purecov: tested */ if (merge_many_buff(info,keys,sort_keys, dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) goto err; /* purecov: inspected */ @@ -259,13 +286,13 @@ err: /* Search after all keys and place them in a temp. file */ -static ha_rows find_all_keys(MI_SORT_PARAM *info, uint keys, +static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek, - int *maxbuffer, IO_CACHE *tempfile, + uint *maxbuffer, IO_CACHE *tempfile, IO_CACHE *tempfile_for_exceptions) { int error; - uint idx; + ha_rows idx; DBUG_ENTER("find_all_keys"); idx=error=0; @@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg) { MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg; int error; - ulonglong memavl, old_memavl; - uint keys, sort_length; - uint idx, maxbuffer; + ulonglong memavl, old_memavl, sortbuff_size; + ha_keys keys, idx; + uint sort_length; + uint maxbuffer; uchar **sort_keys=0; LINT_INIT(keys); @@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg) bzero((char*) &sort_param->unique, sizeof(sort_param->unique)); sort_keys= (uchar **) NULL; - memavl= max(sort_param->sortbuff_size, MIN_SORT_BUFFER); - idx= (uint)sort_param->sort_info->max_records; + sortbuff_size= sort_param->sortbuff_size; + memavl= max(sortbuff_size, MIN_SORT_BUFFER); + idx= (ha_keys) sort_param->sort_info->max_records; sort_length= sort_param->key_length; maxbuffer= 1; - if ((memavl - sizeof(BUFFPEK)) / (sort_length + - sizeof(char *)) > UINT_MAX32) - memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *)); - while (memavl >= MIN_SORT_BUFFER) { if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl) keys= idx+1; + else if ((sort_param->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1); + } else { - uint skr; + uint maxbuffer_org; do { - skr= maxbuffer; + maxbuffer_org= maxbuffer; if (memavl < sizeof(BUFFPEK)*maxbuffer || (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ (sort_length+sizeof(char*))) <= 1 || keys < (uint) maxbuffer) { mi_check_print_error(sort_param->sort_info->param, - "myisam_sort_buffer_size is too small"); + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); goto err; } } - while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr); + while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org); } if ((sort_keys= (uchar**) my_malloc(keys*(sort_length+sizeof(char*))+ @@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg) HA_FT_MAXBYTELEN : 0), MYF(0)))) { if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK), - maxbuffer, maxbuffer/2, MYF(0))) + maxbuffer, min(maxbuffer/2, 1000), MYF(0))) { my_free(sort_keys); sort_keys= (uchar **) NULL; /* for err: label */ @@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg) } if (memavl < MIN_SORT_BUFFER) { + /* purecov: begin inspected */ mi_check_print_error(sort_param->sort_info->param, - "MyISAM sort buffer too small"); - goto err; /* purecov: tested */ + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ } if (sort_param->sort_info->param->testflag & T_VERBOSE) - printf("Key %d - Allocating buffer for %d keys\n", - sort_param->key + 1, keys); + my_fprintf(stdout, + "Key %d - Allocating buffer for %llu keys\n", + sort_param->key + 1, (ulonglong) keys); sort_param->sort_keys= sort_keys; idx= error= 0; @@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg) &sort_param->tempfile)) goto err; sort_keys[0]= (uchar*) (sort_keys+keys); - memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length); + memcpy(sort_keys[0], sort_keys[idx - 1], + (size_t) sort_param->key_length); idx= 1; } sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length; @@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg) else sort_param->keys= idx; - sort_param->sort_keys_length= keys; goto ok; err: @@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) { if (param->testflag & T_VERBOSE) { - printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys); + my_fprintf(stdout, + "Key %d - Dumping %llu keys\n", sinfo->key+1, + (ulonglong) sinfo->keys); fflush(stdout); } if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || @@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) if (maxbuffer >= MERGEBUFF2) { if (param->testflag & T_VERBOSE) - printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); + my_fprintf(stdout, + "Key %d - Merging %llu keys\n", + sinfo->key+1, (ulonglong) sinfo->keys); if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), - (int*) &maxbuffer, &sinfo->tempfile)) + &maxbuffer, &sinfo->tempfile)) { got_error=1; continue; @@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) /* Write all keys in memory to file for later merge */ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, - uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile) { uchar **end; uint sort_length=info->key_length; DBUG_ENTER("write_keys"); + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, info); if (!my_b_inited(tempfile) && @@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, for (end=sort_keys+count ; sort_keys != end ; sort_keys++) { - if (my_b_write(tempfile,(uchar*) *sort_keys,(uint) sort_length)) + if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length)) DBUG_RETURN(1); /* purecov: inspected */ } DBUG_RETURN(0); @@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs) static int write_keys_varlen(MI_SORT_PARAM *info, register uchar **sort_keys, - uint count, BUFFPEK *buffpek, + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile) { uchar **end; int err; DBUG_ENTER("write_keys_varlen"); + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, info); if (!my_b_inited(tempfile) && @@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile) /* Write index */ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys, - register uint count) + register ha_keys count) { DBUG_ENTER("write_index"); @@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys, /* Merge buffers to make < MERGEBUFF2 buffers */ -static int merge_many_buff(MI_SORT_PARAM *info, uint keys, +static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *t_file) + uint *maxbuffer, IO_CACHE *t_file) { - register int i; + register uint i; IO_CACHE t_file2, *from_file, *to_file, *temp; BUFFPEK *lastbuff; DBUG_ENTER("merge_many_buff"); @@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys, reinit_io_cache(from_file,READ_CACHE,0L,0,0); reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); lastbuff=buffpek; - for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) + for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF) { if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, buffpek+i,buffpek+i+MERGEBUFF-1)) @@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys, if (flush_io_cache(to_file)) break; /* purecov: inspected */ temp=from_file; from_file=to_file; to_file=temp; - *maxbuffer= (int) (lastbuff-buffpek)-1; + *maxbuffer= (uint) (lastbuff-buffpek)-1; } cleanup: close_cached_file(to_file); /* This holds old result */ @@ -818,35 +871,36 @@ cleanup: -1 Error */ -static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, - uint sort_length) +static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) { - register uint count; - uint length; + register ha_keys count; + my_off_t length; - if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count))) { if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base, - (length= sort_length*count), + (length= sort_length * count), buffpek->file_pos, MYF_RW)) - return((uint) -1); /* purecov: inspected */ + return(HA_OFFSET_ERROR); /* purecov: inspected */ buffpek->key=buffpek->base; - buffpek->file_pos+= length; /* New filepos */ + buffpek->file_pos+= length; /* New filepos */ buffpek->count-= count; buffpek->mem_count= count; } - return (count*sort_length); + return (((my_off_t) count) * sort_length); } /* read_to_buffer */ -static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, - uint sort_length) + +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) { - register uint count; + register ha_keys count; uint16 length_of_key = 0; uint idx; uchar *buffp; - if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count))) { buffp = buffpek->base; @@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, { if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key, sizeof(length_of_key), buffpek->file_pos, MYF_RW)) - return((uint) -1); + return(HA_OFFSET_ERROR); buffpek->file_pos+=sizeof(length_of_key); if (mysql_file_pread(fromfile->file, (uchar*) buffp, length_of_key, buffpek->file_pos, MYF_RW)) - return((uint) -1); + return(HA_OFFSET_ERROR); buffpek->file_pos+=length_of_key; buffp = buffp + sort_length; } @@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, buffpek->count-= count; buffpek->mem_count= count; } - return (count*sort_length); + return (((my_off_t) count) * sort_length); } /* read_to_buffer_varlen */ static int write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar* key, - uint sort_length, uint count) + uint sort_length, ha_keys count) { - uint idx; + ha_keys idx; uchar *bufs = key; for (idx=1;idx<=count;idx++) @@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info, static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)), IO_CACHE *to_file, uchar *key, - uint sort_length, uint count) + uint sort_length, ha_keys count) { - return my_b_write(to_file, key, (size_t) sort_length*count); + return my_b_write(to_file, key, ((size_t) sort_length) * count); } /* Merge buffers to one buffer If to_file == 0 then use info->key_write + + Return: + 0 ok + 1 error */ static int -merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, +merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb) { - int error; - uint sort_length,maxcount; + int error= 1; + uint sort_length; + ha_keys maxcount; ha_rows count; - my_off_t UNINIT_VAR(to_start_filepos); + my_off_t UNINIT_VAR(to_start_filepos), read_length; uchar *strpos; BUFFPEK *buffpek,**refpek; QUEUE queue; DBUG_ENTER("merge_buffers"); count=error=0; - maxcount=keys/((uint) (Tb-Fb) +1); + maxcount= keys/((uint) (Tb-Fb) +1); DBUG_ASSERT(maxcount > 0); LINT_INIT(to_start_filepos); if (to_file) @@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { count+= buffpek->count; buffpek->base= (uchar*) strpos; - buffpek->max_keys=maxcount; - strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek, - sort_length)); - if (error == -1) + buffpek->max_keys= maxcount; + strpos+= (read_length= info->read_to_buffer(from_file,buffpek, + sort_length)); + if (read_length == HA_OFFSET_ERROR) goto err; /* purecov: inspected */ queue_insert(&queue,(uchar*) buffpek); } @@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, /* It's enough to check for killedptr before a slow operation */ if (killed_ptr(info->sort_info->param)) { - error=1; goto err; } - if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length))) + if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length))) { uchar *base= buffpek->base; uint max_keys=buffpek->max_keys; @@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } break; /* One buffer have been removed */ } + else if (read_length == HA_OFFSET_ERROR) + goto err; /* purecov: inspected */ } - else if (error == -1) - goto err; /* purecov: inspected */ queue_replace_top(&queue); /* Top element has been replaced */ } } @@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } } } - while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 && - error != 0); + while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0); + if (read_length == 0) + error= 0; lastbuff->count=count; if (to_file) @@ -1045,8 +1104,8 @@ err: /* Do a merge to output-file (save only positions) */ static int -merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, - BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) +merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, + BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile) { DBUG_ENTER("merge_index"); if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, @@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, DBUG_RETURN(0); } /* merge_index */ + static int flush_ft_buf(MI_SORT_PARAM *info) { @@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info) } return err; } - |