diff options
Diffstat (limited to 'myisam')
63 files changed, 5841 insertions, 2661 deletions
diff --git a/myisam/Makefile.am b/myisam/Makefile.am index b68126d5d69..aeb3b34ee15 100644 --- a/myisam/Makefile.am +++ b/myisam/Makefile.am @@ -1,15 +1,15 @@ # Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA @@ -17,7 +17,7 @@ EXTRA_DIST = mi_test_all.sh mi_test_all.res pkgdata_DATA = mi_test_all mi_test_all.res -INCLUDES = @MT_INCLUDES@ -I$(srcdir)/../include -I../include +INCLUDES = @MT_INCLUDES@ -I$(srcdir)/../include -I../include LDADD = @CLIENT_EXTRA_LDFLAGS@ libmyisam.a ../mysys/libmysys.a \ ../dbug/libdbug.a ../strings/libmystrings.a pkglib_LIBRARIES = libmyisam.a @@ -25,13 +25,14 @@ bin_PROGRAMS = myisamchk myisamlog myisampack myisamchk_DEPENDENCIES= $(LIBRARIES) myisamlog_DEPENDENCIES= $(LIBRARIES) myisampack_DEPENDENCIES=$(LIBRARIES) -noinst_PROGRAMS = mi_test1 mi_test2 mi_test3 ft_test1 ft_eval +noinst_PROGRAMS = mi_test1 mi_test2 mi_test3 ft_dump #ft_test1 ft_eval noinst_HEADERS = myisamdef.h fulltext.h ftdefs.h ft_test1.h ft_eval.h mi_test1_DEPENDENCIES= $(LIBRARIES) mi_test2_DEPENDENCIES= $(LIBRARIES) mi_test3_DEPENDENCIES= $(LIBRARIES) -ft_test1_DEPENDENCIES= $(LIBRARIES) -ft_eval_DEPENDENCIES= $(LIBRARIES) +#ft_test1_DEPENDENCIES= $(LIBRARIES) +#ft_eval_DEPENDENCIES= $(LIBRARIES) +ft_dump_DEPENDENCIES= $(LIBRARIES) libmyisam_a_SOURCES = mi_open.c mi_extra.c mi_info.c mi_rkey.c \ mi_rnext.c mi_rnext_same.c \ mi_search.c mi_page.c mi_key.c mi_locking.c \ @@ -44,8 +45,8 @@ libmyisam_a_SOURCES = mi_open.c mi_extra.c mi_info.c mi_rkey.c \ mi_range.c mi_dbug.c mi_checksum.c mi_log.c \ mi_changed.c mi_static.c mi_delete_all.c \ mi_delete_table.c mi_rename.c mi_check.c \ - ft_parser.c ft_search.c ft_stopwords.c ft_static.c \ - ft_update.c sort.c + ft_parser.c ft_stopwords.c ft_static.c \ + ft_update.c ft_boolean_search.c ft_nlq_search.c sort.c CLEANFILES = test?.MY? FT?.MY? isam.log mi_test_all DEFS = -DMAP_TO_USE_RAID diff --git a/myisam/ft_boolean_search.c b/myisam/ft_boolean_search.c new file mode 100644 index 00000000000..97c55c1d937 --- /dev/null +++ b/myisam/ft_boolean_search.c @@ -0,0 +1,635 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* TODO: add caching - pre-read several index entries at once */ + +#define FT_CORE +#include "ftdefs.h" +#include <queues.h> + +/* search with boolean queries */ + +static double _wghts[11]= +{ + 0.131687242798354, + 0.197530864197531, + 0.296296296296296, + 0.444444444444444, + 0.666666666666667, + 1.000000000000000, + 1.500000000000000, + 2.250000000000000, + 3.375000000000000, + 5.062500000000000, + 7.593750000000000}; +static double *wghts=_wghts+5; /* wghts[i] = 1.5**i */ + +static double _nwghts[11]= +{ + -0.065843621399177, + -0.098765432098766, + -0.148148148148148, + -0.222222222222222, + -0.333333333333334, + -0.500000000000000, + -0.750000000000000, + -1.125000000000000, + -1.687500000000000, + -2.531250000000000, + -3.796875000000000}; +static double *nwghts=_nwghts+5; /* nwghts[i] = -0.5*1.5**i */ + +#define FTB_FLAG_TRUNC 1 /* MUST be 1 */ +#define FTB_FLAG_YES 2 /* These two - YES and NO */ +#define FTB_FLAG_NO 4 /* should NEVER be set both */ + +typedef struct st_ftb_expr FTB_EXPR; +struct st_ftb_expr +{ + FTB_EXPR *up; + byte *quot, *qend; + float weight; + uint flags; + my_off_t docid[2]; /* for index search and for scan */ + float cur_weight; + int yesses; /* number of "yes" words matched */ + int nos; /* number of "no" words matched */ + int ythresh; /* number of "yes" words in expr */ + int yweaks; /* number of "yes" words for scan only */ +}; + +typedef struct st_ftb_word +{ + FTB_EXPR *up; + float weight; + uint flags; + my_off_t docid[2]; /* for index search and for scan */ + uint ndepth; + int len; + /* ... docid cache can be added here. SerG */ + byte word[1]; +} FTB_WORD; + +typedef struct st_ft_info +{ + struct _ft_vft *please; + MI_INFO *info; + uint keynr; + CHARSET_INFO *charset; + enum { UNINITIALIZED, READY, INDEX_SEARCH, INDEX_DONE /*, SCAN*/ } state; + uint with_scan; + my_off_t lastpos; + FTB_EXPR *root; + QUEUE queue; + TREE no_dupes; + FTB_WORD **list; + MEM_ROOT mem_root; +} FTB; + +static int FTB_WORD_cmp(my_off_t *v, FTB_WORD *a, FTB_WORD *b) +{ + int i; + + /* if a==curdoc, take it as a < b */ + if (v && a->docid[0] == *v) + return -1; + + /* ORDER BY docid, ndepth DESC */ + i=CMP_NUM(a->docid[0], b->docid[0]); + if (!i) + i=CMP_NUM(b->ndepth,a->ndepth); + return i; +} + +static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b) +{ + /* ORDER BY word DESC, ndepth DESC */ + int i=_mi_compare_text(cs, (uchar*) (*b)->word+1,(*b)->len-1, + (uchar*) (*a)->word+1,(*a)->len-1,0); + if (!i) + i=CMP_NUM((*b)->ndepth,(*a)->ndepth); + return i; +} + +static void _ftb_parse_query(FTB *ftb, byte **start, byte *end, + FTB_EXPR *up, uint depth) +{ + byte res; + FTB_PARAM param; + FT_WORD w; + FTB_WORD *ftbw; + FTB_EXPR *ftbe; + uint extra=HA_FT_WLEN+ftb->info->s->rec_reflength; /* just a shortcut */ + + if (ftb->state != UNINITIALIZED) + return; + + param.prev=' '; + param.quot=up->quot; + while ((res=ft_get_word(start,end,&w,¶m))) + { + int r=param.plusminus; + float weight= (float) (param.pmsign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)]; + switch (res) { + case 1: /* word found */ + ftbw=(FTB_WORD *)alloc_root(&ftb->mem_root, + sizeof(FTB_WORD) + + (param.trunc ? MI_MAX_KEY_BUFF : + w.len+extra)); + ftbw->len=w.len+1; + ftbw->flags=0; + if (param.yesno>0) ftbw->flags|=FTB_FLAG_YES; + if (param.yesno<0) ftbw->flags|=FTB_FLAG_NO; + if (param.trunc) ftbw->flags|=FTB_FLAG_TRUNC; + ftbw->weight=weight; + ftbw->up=up; + ftbw->docid[0]=ftbw->docid[1]=HA_POS_ERROR; + ftbw->ndepth= (param.yesno<0) + depth; + memcpy(ftbw->word+1, w.pos, w.len); + ftbw->word[0]=w.len; + if (param.yesno > 0) up->ythresh++; + queue_insert(& ftb->queue, (byte *)ftbw); + ftb->with_scan|=(param.trunc & FTB_FLAG_TRUNC); + break; + case 2: /* left bracket */ + ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR)); + ftbe->flags=0; + if (param.yesno>0) ftbe->flags|=FTB_FLAG_YES; + if (param.yesno<0) ftbe->flags|=FTB_FLAG_NO; + ftbe->weight=weight; + ftbe->up=up; + ftbe->ythresh=ftbe->yweaks=0; + ftbe->docid[0]=ftbe->docid[1]=HA_POS_ERROR; + if ((ftbe->quot=param.quot)) ftb->with_scan|=2; + if (param.yesno > 0) up->ythresh++; + _ftb_parse_query(ftb, start, end, ftbe, depth+1); + param.quot=0; + break; + case 3: /* right bracket */ + if (up->quot) up->qend=param.quot; + return; + } + } + return; +} + +static int _ftb_no_dupes_cmp(void* not_used __attribute__((unused)), + const void *a,const void *b) +{ + return CMP_NUM((*((my_off_t*)a)), (*((my_off_t*)b))); +} + +static void _ftb_init_index_search(FT_INFO *ftb) +{ + int i, r; + FTB_WORD *ftbw; + MI_INFO *info=ftb->info; + MI_KEYDEF *keyinfo; + my_off_t keyroot; + + if ((ftb->state != READY && ftb->state !=INDEX_DONE) || + ftb->keynr == NO_SUCH_KEY) + return; + ftb->state=INDEX_SEARCH; + + keyinfo=info->s->keyinfo+ftb->keynr; + keyroot=info->s->state.key_root[ftb->keynr]; + + for (i=ftb->queue.elements; i; i--) + { + ftbw=(FTB_WORD *)(ftb->queue.root[i]); + + if (ftbw->flags & FTB_FLAG_TRUNC) + { + /* + special treatment for truncation operator :(( + 1. +trunc* and there're other (not +trunc*) words + | no need to search in the index, it can never ADD new rows + | to the result, and to remove half-matched rows we do scan anyway + 2. -trunc* + | same as 1. + 3. trunc* + | We have to index-search for this prefix. + | It may cause duplicates, as in the index (sorted by <word,docid>) + | <aaaa,row1> + | <aabb,row2> + | <aacc,row1> + | Searching for "aa*" will find row1 twice... + */ + if ( test(ftbw->flags&FTB_FLAG_NO) || /* 2 */ + (test(ftbw->flags&FTB_FLAG_YES) && /* 1 */ + ftbw->up->ythresh - ftbw->up->yweaks >1)) /* 1 */ + { + ftbw->docid[0]=HA_POS_ERROR; + ftbw->up->yweaks++; + continue; + } + else /* 3 */ + { + if (!is_tree_inited(& ftb->no_dupes)) + { + init_tree(& ftb->no_dupes,0,0,sizeof(my_off_t), + _ftb_no_dupes_cmp,0,0,0); + } + } + } + r=_mi_search(info, keyinfo, (uchar*) ftbw->word, ftbw->len, + SEARCH_FIND | SEARCH_BIGGER, keyroot); + if (!r) + { + r=_mi_compare_text(ftb->charset, + info->lastkey + (ftbw->flags&FTB_FLAG_TRUNC), + ftbw->len - (ftbw->flags&FTB_FLAG_TRUNC), + (uchar*) ftbw->word + (ftbw->flags&FTB_FLAG_TRUNC), + ftbw->len - (ftbw->flags&FTB_FLAG_TRUNC), + 0); + } + if (r) /* not found */ + { + if (ftbw->flags&FTB_FLAG_YES && ftbw->up->up==0) + { + /* + This word MUST BE present in every document returned, + so we can abort the search right now + */ + ftb->state=INDEX_DONE; + return; + } + } + else + { + memcpy(ftbw->word, info->lastkey, info->lastkey_length); + ftbw->docid[0]=info->lastpos; + } + } + queue_fix(& ftb->queue); +} + + +FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query, + uint query_len, + my_bool presort __attribute__((unused))) +{ + FTB *ftb; + FTB_EXPR *ftbe; + uint res; + + if (!(ftb=(FTB *)my_malloc(sizeof(FTB), MYF(MY_WME)))) + return 0; + ftb->please= (struct _ft_vft *) & _ft_vft_boolean; + ftb->state=UNINITIALIZED; + ftb->info=info; + ftb->keynr=keynr; + ftb->charset= ((keynr==NO_SUCH_KEY) ? + default_charset_info : + info->s->keyinfo[keynr].seg->charset); + ftb->with_scan=0; + ftb->lastpos=0; + bzero(& ftb->no_dupes, sizeof(TREE)); + + init_alloc_root(&ftb->mem_root, 1024, 1024); + + /* + Hack: instead of init_queue, we'll use reinit queue to be able + to alloc queue with alloc_root() + */ + res=ftb->queue.max_elements=1+query_len/(ft_min_word_len+1); + ftb->queue.root=(byte **)alloc_root(&ftb->mem_root, (res+1)*sizeof(void*)); + reinit_queue(& ftb->queue, res, 0, 0, + (int (*)(void*,byte*,byte*))FTB_WORD_cmp, 0); + ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR)); + ftbe->weight=1; + ftbe->flags=FTB_FLAG_YES; + ftbe->nos=1; + ftbe->quot=0; + ftbe->up=0; + ftbe->ythresh=ftbe->yweaks=0; + ftbe->docid[0]=ftbe->docid[1]=HA_POS_ERROR; + ftb->root=ftbe; + _ftb_parse_query(ftb, &query, query+query_len, ftbe, 0); + ftb->list=(FTB_WORD **)alloc_root(&ftb->mem_root, + sizeof(FTB_WORD *)*ftb->queue.elements); + memcpy(ftb->list, ftb->queue.root+1, sizeof(FTB_WORD *)*ftb->queue.elements); + qsort2(ftb->list, ftb->queue.elements, sizeof(FTB_WORD *), + (qsort2_cmp)FTB_WORD_cmp_list, ftb->charset); + if (ftb->queue.elements<2) ftb->with_scan &= ~FTB_FLAG_TRUNC; + ftb->state=READY; + return ftb; +} + + +/* returns 1 if str0 contain str1 */ +static int _ftb_strstr(const byte *s0, const byte *e0, + const byte *s1, const byte *e1, + CHARSET_INFO *cs) +{ + const byte *p; + + while (s0 < e0) + { + while (s0 < e0 && cs->to_upper[(uint) (uchar) *s0++] != + cs->to_upper[(uint) (uchar) *s1]) + /* no-op */; + if (s0 >= e0) + return 0; + p=s1+1; + while (s0 < e0 && p < e1 && cs->to_upper[(uint) (uchar) *s0] == + cs->to_upper[(uint) (uchar) *p]) + s0++, p++; + if (p >= e1) + return 1; + } + return 0; +} + + +static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_orig) +{ + FT_SEG_ITERATOR ftsi; + FTB_EXPR *ftbe; + float weight=ftbw->weight; + int yn=ftbw->flags, ythresh, mode=(ftsi_orig != 0); + my_off_t curdoc=ftbw->docid[mode]; + + for (ftbe=ftbw->up; ftbe; ftbe=ftbe->up) + { + ythresh = ftbe->ythresh - (mode ? 0 : ftbe->yweaks); + if (ftbe->docid[mode] != curdoc) + { + ftbe->cur_weight=0; + ftbe->yesses=ftbe->nos=0; + ftbe->docid[mode]=curdoc; + } + if (ftbe->nos) + break; + if (yn & FTB_FLAG_YES) + { + weight /= ftbe->ythresh; + ftbe->cur_weight += weight; + if (++ftbe->yesses == ythresh) + { + yn=ftbe->flags; + weight=ftbe->cur_weight*ftbe->weight; + if (mode && ftbe->quot) + { + int not_found=1; + + memcpy(&ftsi, ftsi_orig, sizeof(ftsi)); + while (_mi_ft_segiterator(&ftsi) && not_found) + { + if (!ftsi.pos) + continue; + not_found = ! _ftb_strstr(ftsi.pos, ftsi.pos+ftsi.len, + ftbe->quot, ftbe->qend, ftb->charset); + } + if (not_found) break; + } /* ftbe->quot */ + } + else + break; + } + else + if (yn & FTB_FLAG_NO) + { + /* + NOTE: special sort function of queue assures that all + (yn & FTB_FLAG_NO) != 0 + events for every particular subexpression will + "auto-magically" happen BEFORE all the + (yn & FTB_FLAG_YES) != 0 events. So no + already matched expression can become not-matched again. + */ + ++ftbe->nos; + break; + } + else + { + if (ftbe->ythresh) + weight/=3; + ftbe->cur_weight += weight; + if (ftbe->yesses < ythresh) + break; + yn= (ftbe->yesses++ == ythresh) ? ftbe->flags : 0 ; + weight*= ftbe->weight; + } + } +} + + +int ft_boolean_read_next(FT_INFO *ftb, char *record) +{ + FTB_EXPR *ftbe; + FTB_WORD *ftbw; + MI_INFO *info=ftb->info; + MI_KEYDEF *keyinfo=info->s->keyinfo+ftb->keynr; + my_off_t keyroot=info->s->state.key_root[ftb->keynr]; + my_off_t curdoc; + int r; + + if (ftb->state != INDEX_SEARCH && ftb->state != INDEX_DONE) + return -1; + + /* black magic ON */ + if ((int) _mi_check_index(info, ftb->keynr) < 0) + return my_errno; + if (_mi_readinfo(info, F_RDLCK, 1)) + return my_errno; + /* black magic OFF */ + + if (!ftb->queue.elements) + return my_errno=HA_ERR_END_OF_FILE; + + /* Attention!!! Address of a local variable is used here! See err: label */ + ftb->queue.first_cmp_arg=(void *)&curdoc; + + while (ftb->state == INDEX_SEARCH && + (curdoc=((FTB_WORD *)queue_top(& ftb->queue))->docid[0]) != + HA_POS_ERROR) + { + while (curdoc==(ftbw=(FTB_WORD *)queue_top(& ftb->queue))->docid[0]) + { + _ftb_climb_the_tree(ftb, ftbw, 0); + + /* update queue */ + r=_mi_search(info, keyinfo, (uchar*) ftbw->word, USE_WHOLE_KEY, + SEARCH_BIGGER , keyroot); + if (!r) + { + r=_mi_compare_text(ftb->charset, + info->lastkey + (ftbw->flags&FTB_FLAG_TRUNC), + ftbw->len - (ftbw->flags&FTB_FLAG_TRUNC), + (uchar*) ftbw->word + (ftbw->flags&FTB_FLAG_TRUNC), + ftbw->len - (ftbw->flags&FTB_FLAG_TRUNC), + 0); + } + if (r) /* not found */ + { + ftbw->docid[0]=HA_POS_ERROR; + if (ftbw->flags&FTB_FLAG_YES && ftbw->up->up==0) + { + /* + This word MUST BE present in every document returned, + so we can stop the search right now + */ + ftb->state=INDEX_DONE; + } + } + else + { + memcpy(ftbw->word, info->lastkey, info->lastkey_length); + ftbw->docid[0]=info->lastpos; + } + queue_replaced(& ftb->queue); + } + + ftbe=ftb->root; + if (ftbe->docid[0]==curdoc && ftbe->cur_weight>0 && + ftbe->yesses>=(ftbe->ythresh-ftbe->yweaks) && !ftbe->nos) + { + /* curdoc matched ! */ + if (is_tree_inited(& ftb->no_dupes) && + tree_insert(& ftb->no_dupes, &curdoc, 0)->count >1) + /* but it managed to get past this line once */ + continue; + + info->lastpos=curdoc; + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (!(*info->read_record)(info,curdoc,record)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + if (ftb->with_scan && ft_boolean_find_relevance(ftb,record,0)==0) + continue; /* no match */ + my_errno=0; + goto err; + } + goto err; + } + } + ftb->state=INDEX_DONE; + my_errno=HA_ERR_END_OF_FILE; +err: + ftb->queue.first_cmp_arg=(void *)0; + return my_errno; +} + + +float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length) +{ + FT_WORD word; + FTB_WORD *ftbw; + FTB_EXPR *ftbe; + FT_SEG_ITERATOR ftsi, ftsi2; + const byte *end; + my_off_t docid=ftb->info->lastpos; + + if (docid == HA_POS_ERROR) + return -2.0; + if (!ftb->queue.elements) + return 0; + + if (ftb->state != INDEX_SEARCH && docid < ftb->lastpos) + { + FTB_EXPR *x; + uint i; + + for (i=0; i < ftb->queue.elements; i++) + { + ftb->list[i]->docid[1]=HA_POS_ERROR; + for (x=ftb->list[i]->up; x; x=x->up) + x->docid[1]=HA_POS_ERROR; + } + } + + ftb->lastpos=docid; + + if (ftb->keynr==NO_SUCH_KEY) + _mi_ft_segiterator_dummy_init(record, length, &ftsi); + else + _mi_ft_segiterator_init(ftb->info, ftb->keynr, record, &ftsi); + memcpy(&ftsi2, &ftsi, sizeof(ftsi)); + + while (_mi_ft_segiterator(&ftsi)) + { + if (!ftsi.pos) + continue; + + end=ftsi.pos+ftsi.len; + while (ft_simple_get_word((byte **) &ftsi.pos,(byte *) end, &word)) + { + int a, b, c; + for (a=0, b=ftb->queue.elements, c=(a+b)/2; b-a>1; c=(a+b)/2) + { + ftbw=ftb->list[c]; + if (_mi_compare_text(ftb->charset, (uchar*) word.pos, word.len, + (uchar*) ftbw->word+1, ftbw->len-1, + (my_bool) (ftbw->flags&FTB_FLAG_TRUNC)) >0) + b=c; + else + a=c; + } + for (; c>=0; c--) + { + ftbw=ftb->list[c]; + if (_mi_compare_text(ftb->charset, (uchar*) word.pos, word.len, + (uchar*) ftbw->word+1,ftbw->len-1, + (my_bool) (ftbw->flags&FTB_FLAG_TRUNC))) + break; + if (ftbw->docid[1] == docid) + continue; + ftbw->docid[1]=docid; + _ftb_climb_the_tree(ftb, ftbw, &ftsi2); + } + } + } + + ftbe=ftb->root; + if (ftbe->docid[1]==docid && ftbe->cur_weight>0 && + ftbe->yesses>=ftbe->ythresh && !ftbe->nos) + { /* row matched ! */ + return ftbe->cur_weight; + } + else + { /* match failed ! */ + return 0.0; + } +} + + +void ft_boolean_close_search(FT_INFO *ftb) +{ + if (is_tree_inited(& ftb->no_dupes)) + { + delete_tree(& ftb->no_dupes); + } + free_root(& ftb->mem_root, MYF(0)); + my_free((gptr)ftb,MYF(0)); +} + + +float ft_boolean_get_relevance(FT_INFO *ftb) +{ + return ftb->root->cur_weight; +} + + +void ft_boolean_reinit_search(FT_INFO *ftb) +{ + _ftb_init_index_search(ftb); +} + diff --git a/myisam/ft_dump.c b/myisam/ft_dump.c new file mode 100644 index 00000000000..d95e719e234 --- /dev/null +++ b/myisam/ft_dump.c @@ -0,0 +1,270 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code + added support for long options (my_getopt) 22.5.2002 by Jani Tolonen */ + +#include "ftdefs.h" +#include <my_getopt.h> + +static void get_options(int *argc,char **argv[]); +static void usage(); +static void complain(int val); + +static int count=0, stats=0, dump=0, lstats=0; +static my_bool verbose; +static char *query=NULL; +static uint lengths[256]; + +#define MAX_LEN (HA_FT_MAXLEN+10) +#define HOW_OFTEN_TO_WRITE 10000 + +static struct my_option my_long_options[] = +{ + {"dump", 'd', "Dump index (incl. data offsets and word weights)", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"stats", 's', "Report global stats", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Be verbose", + (gptr*) &verbose, (gptr*) &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"count", 'c', "Calculate per-word stats (counts and global weights)", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"length", 'l', "Report length distribution", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"execute", 'e', "Execute given query", (gptr*) &query, (gptr*) &query, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"help", 'h', "Display help and exit", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"help", '?', "Synonym for -h", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +int main(int argc,char *argv[]) +{ + int error=0; + uint keylen, keylen2=0, inx, doc_cnt=0; + float weight; + double gws, min_gws=0, avg_gws=0; + MI_INFO *info; + char buf[MAX_LEN], buf2[MAX_LEN], buf_maxlen[MAX_LEN], buf_min_gws[MAX_LEN]; + ulong total=0, maxlen=0, uniq=0, max_doc_cnt=0; + struct { MI_INFO *info; } aio0, *aio=&aio0; /* for GWS_IN_USE */ + + MY_INIT(argv[0]); + get_options(&argc, &argv); + if (count || dump) + verbose=0; + if (!count && !dump && !lstats && !query) + stats=1; + + if (verbose) + setbuf(stdout,NULL); + + if (argc < 2) + usage(); + + if (!(info=mi_open(argv[0],2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + + inx=atoi(argv[1]); + *buf2=0; + aio->info=info; + + if ((inx >= info->s->base.keys) || + !(info->s->keyinfo[inx].flag & HA_FULLTEXT)) + { + printf("Key %d in table %s is not a FULLTEXT key\n", inx, info->filename); + goto err; + } + + if (query) + { +#if 0 + FT_DOCLIST *result; + int i; + + ft_init_stopwords(ft_precompiled_stopwords); + + result=ft_nlq_init_search(info,inx,query,strlen(query),1); + if(!result) + goto err; + + if (verbose) + printf("%d rows matched\n",result->ndocs); + + for(i=0 ; i<result->ndocs ; i++) + printf("%9qx %20.7f\n",result->doc[i].dpos,result->doc[i].weight); + + ft_nlq_close_search(result); +#else + printf("-e option is disabled\n"); +#endif + } + else + { + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_PREV_FOUND; + + while (!(error=mi_rnext(info,NULL,inx))) + { + keylen=*(info->lastkey); + +#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT + mi_float4get(weight,info->lastkey+keylen+1); +#else +#error +#endif + + snprintf(buf,MAX_LEN,"%.*s",(int) keylen,info->lastkey+1); + casedn_str(buf); + total++; + lengths[keylen]++; + + if (count || stats) + { + doc_cnt++; + if (strcmp(buf, buf2)) + { + if (*buf2) + { + uniq++; + avg_gws+=gws=GWS_IN_USE; + if (count) + printf("%9u %20.7f %s\n",doc_cnt,gws,buf2); + if (maxlen<keylen2) + { + maxlen=keylen2; + strmov(buf_maxlen, buf2); + } + if (max_doc_cnt < doc_cnt) + { + max_doc_cnt=doc_cnt; + strmov(buf_min_gws, buf2); + min_gws=gws; + } + } + strmov(buf2, buf); + keylen2=keylen; + doc_cnt=0; + } + } + if (dump) + printf("%9qx %20.7f %s\n",info->lastpos,weight,buf); + + if(verbose && (total%HOW_OFTEN_TO_WRITE)==0) + printf("%10ld\r",total); + } + + if (stats) + { + count=0; + for (inx=0;inx<256;inx++) + { + count+=lengths[inx]; + if ((ulong) count >= total/2) + break; + } + printf("Total rows: %qu\nTotal words: %lu\n" + "Unique words: %lu\nLongest word: %lu chars (%s)\n" + "Median length: %u\n" + "Average global weight: %f\n" + "Most common word: %lu times, weight: %f (%s)\n", + (ulonglong)info->state->records, total, uniq, maxlen, buf_maxlen, + inx, avg_gws/uniq, max_doc_cnt, min_gws, buf_min_gws); + } + if (lstats) + { + count=0; + for (inx=0; inx<256; inx++) + { + count+=lengths[inx]; + if (count && lengths[inx]) + printf("%3u: %10lu %5.2f%% %20lu %4.1f%%\n", inx, + (ulong) lengths[inx],100.0*lengths[inx]/total,(ulong) count, + 100.0*count/total); + } + } + } + +err: + if (error && error != HA_ERR_END_OF_FILE) + printf("got error %d\n",my_errno); + if (info) + mi_close(info); + return 0; +} + + +static my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch(optid) { + case 'd': + dump=1; + complain(count || query); + break; + case 's': + stats=1; + complain(query!=0); + break; + case 'c': + count= 1; + complain(dump || query); + break; + case 'l': + lstats=1; + complain(query!=0); + break; + case 'e': + complain(dump || count || stats); + break; + case '?': + case 'h': + usage(); + } + return 0; +} + + +static void get_options(int *argc, char **argv[]) +{ + int ho_error; + + if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option))) + exit(ho_error); +} /* get options */ + + +static void usage() +{ + printf("Use: ft_dump <table_name> <index_no>\n"); + my_print_help(my_long_options); + my_print_variables(my_long_options); + exit(1); +} + + +static void complain(int val) /* Kinda assert :-) */ +{ + if (val) + { + printf("You cannot use these options together!\n"); + exit(1); + } +} diff --git a/myisam/ft_eval.c b/myisam/ft_eval.c index 9466104100a..34248c69f20 100644 --- a/myisam/ft_eval.c +++ b/myisam/ft_eval.c @@ -11,18 +11,32 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ +/* Written by Sergei A. Golubchik, who has a shared copyright to this code + added support for long options (my_getopt) 22.5.2002 by Jani Tolonen */ #include "ftdefs.h" #include "ft_eval.h" #include <stdarg.h> -#include <getopt.h> +#include <my_getopt.h> static void print_error(int exit_code, const char *fmt,...); static void get_options(int argc, char *argv[]); static int create_record(char *pos, FILE *file); +static void usage(); -int main(int argc,char *argv[]) +static struct my_option my_long_options[] = +{ + {"", 's', "", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'q', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'S', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", '#', "", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'V', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", '?', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'h', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +int main(int argc, char *argv[]) { MI_INFO *file; int i,j; @@ -32,7 +46,7 @@ int main(int argc,char *argv[]) bzero((char*)recinfo,sizeof(recinfo)); /* First define 2 columns */ - recinfo[0].type=FIELD_SKIPP_ENDSPACE; + recinfo[0].type=FIELD_SKIP_ENDSPACE; recinfo[0].length=docid_length; recinfo[1].type=FIELD_BLOB; recinfo[1].length= 4+mi_portable_sizeof_char_ptr; @@ -65,7 +79,7 @@ int main(int argc,char *argv[]) my_errno=0; i=0; - while(create_record(record,df)) + while (create_record(record,df)) { error=mi_write(file,record); if (error) @@ -80,102 +94,125 @@ int main(int argc,char *argv[]) if (!(file=mi_open(filename,2,0))) goto err; if (!silent) printf("- Reading rows with key\n"); - for(i=1;create_record(record,qf);i++) { - FT_DOCLIST *result; double w; int t,err; + for (i=1;create_record(record,qf);i++) + { + FT_DOCLIST *result; + double w; + int t, err; - result=ft_init_search(file,0,blob_record,(uint) strlen(blob_record),1); - if(!result) { + result=ft_nlq_init_search(file,0,blob_record,(uint) strlen(blob_record),1); + if (!result) + { printf("Query %d failed with errno %3d\n",i,my_errno); goto err; } if (!silent) printf("Query %d. Found: %d.\n",i,result->ndocs); - for(j=0;(err=ft_read_next(result, read_record))==0;j++) { + for (j=0;(err=ft_nlq_read_next(result, read_record))==0;j++) + { t=uint2korr(read_record); - w=ft_get_relevance(result); + w=ft_nlq_get_relevance(result); printf("%d %.*s %f\n",i,t,read_record+2,w); } - if(err != HA_ERR_END_OF_FILE) { + if (err != HA_ERR_END_OF_FILE) + { printf("ft_read_next %d failed with errno %3d\n",j,my_errno); goto err; } - ft_close_search(result); + ft_nlq_close_search(result); } if (mi_close(file)) goto err; my_end(MY_CHECK_ERROR); return (0); -err: + + err: printf("got error: %3d when using myisam-database\n",my_errno); - return 1; /* skipp warning */ + return 1; /* skip warning */ } -static void get_options(int argc,char *argv[]) -{ - int c; - char *options=(char*) "Vh#:qSs:"; - while ((c=getopt(argc,argv,options)) != -1) - { - switch(c) { - case 's': - if(stopwordlist && stopwordlist!=ft_precompiled_stopwords) break; +static my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch (optid) { + case 's': + if (stopwordlist && stopwordlist != ft_precompiled_stopwords) + break; + { + FILE *f; char s[HA_FT_MAXLEN]; int i=0,n=SWL_INIT; + + if (!(stopwordlist=(const char**) malloc(n*sizeof(char *)))) + print_error(1,"malloc(%d)",n*sizeof(char *)); + if (!(f=fopen(argument,"r"))) + print_error(1,"fopen(%s)",argument); + while (!feof(f)) { - FILE *f; char s[HA_FT_MAXLEN]; int i=0,n=SWL_INIT; - - if(!(stopwordlist=(const char**) malloc(n*sizeof(char *)))) - print_error(1,"malloc(%d)",n*sizeof(char *)); - if(!(f=fopen(optarg,"r"))) - print_error(1,"fopen(%s)",optarg); - while(!feof(f)) { - if(!(fgets(s,HA_FT_MAXLEN,f))) - print_error(1,"fgets(s,%d,%s)",HA_FT_MAXLEN,optarg); - if(!(stopwordlist[i++]=strdup(s))) - print_error(1,"strdup(%s)",s); - if(i>=n) { - n+=SWL_PLUS; - if(!(stopwordlist=(const char**) realloc((char*) stopwordlist,n*sizeof(char *)))) - print_error(1,"realloc(%d)",n*sizeof(char *)); - } + if (!(fgets(s,HA_FT_MAXLEN,f))) + print_error(1,"fgets(s,%d,%s)",HA_FT_MAXLEN,argument); + if (!(stopwordlist[i++]=strdup(s))) + print_error(1,"strdup(%s)",s); + if (i >= n) + { + n+=SWL_PLUS; + if (!(stopwordlist=(const char**) realloc((char*) stopwordlist, + n*sizeof(char *)))) + print_error(1,"realloc(%d)",n*sizeof(char *)); } - fclose(f); - stopwordlist[i]=NULL; - break; } - case 'q': silent=1; break; - case 'S': if(stopwordlist==ft_precompiled_stopwords) stopwordlist=NULL; break; - case '#': - DEBUGGER_ON; - DBUG_PUSH (optarg); + fclose(f); + stopwordlist[i]=NULL; break; - case 'V': - case '?': - case 'h': - default: - printf("%s -[%s] <d_file> <q_file>\n", argv[0], options); - exit(0); } + case 'q': silent=1; break; + case 'S': if (stopwordlist==ft_precompiled_stopwords) stopwordlist=NULL; break; + case '#': + DEBUGGER_ON; + DBUG_PUSH (argument); + break; + case 'V': + case '?': + case 'h': + usage(); + exit(1); } - if(!(d_file=argv[optind])) print_error(1,"No d_file"); - if(!(df=fopen(d_file,"r"))) + return 0; +} + + +static void get_options(int argc, char *argv[]) +{ + int ho_error; + + if ((ho_error=handle_options(&argc, &argv, my_long_options, get_one_option))) + exit(ho_error); + + if (!(d_file=argv[optind])) print_error(1,"No d_file"); + if (!(df=fopen(d_file,"r"))) print_error(1,"fopen(%s)",d_file); - if(!(q_file=argv[optind+1])) print_error(1,"No q_file"); - if(!(qf=fopen(q_file,"r"))) + if (!(q_file=argv[optind+1])) print_error(1,"No q_file"); + if (!(qf=fopen(q_file,"r"))) print_error(1,"fopen(%s)",q_file); return; } /* get options */ + static int create_record(char *pos, FILE *file) -{ uint tmp; char *ptr; +{ + uint tmp; char *ptr; bzero((char *)pos,MAX_REC_LENGTH); /* column 1 - VARCHAR */ - if(!(fgets(pos+2,MAX_REC_LENGTH-32,file))) + if (!(fgets(pos+2,MAX_REC_LENGTH-32,file))) { - if(feof(file)) return 0; else print_error(1,"fgets(docid) - 1"); + if (feof(file)) + return 0; + else + print_error(1,"fgets(docid) - 1"); } tmp=(uint) strlen(pos+2)-1; int2store(pos,tmp); @@ -183,7 +220,7 @@ static int create_record(char *pos, FILE *file) /* column 2 - BLOB */ - if(!(fgets(blob_record,MAX_BLOB_LENGTH,file))) + if (!(fgets(blob_record,MAX_BLOB_LENGTH,file))) print_error(1,"fgets(docid) - 2"); tmp=(uint) strlen(blob_record); int4store(pos,tmp); @@ -206,3 +243,11 @@ static void print_error(int exit_code, const char *fmt,...) va_end(args); exit(exit_code); } + + +static void usage() +{ + printf("%s [options]\n", my_progname); + my_print_help(my_long_options); + my_print_variables(my_long_options); +} diff --git a/myisam/ft_eval.h b/myisam/ft_eval.h index d87b6be9c7c..68be3a39f33 100644 --- a/myisam/ft_eval.h +++ b/myisam/ft_eval.h @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & Sergei A. Golubchik - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -40,4 +40,3 @@ MI_KEYSEG keyseg[10]; #define MAX_LINE_LENGTH 128 char line[MAX_LINE_LENGTH]; - diff --git a/myisam/ft_search.c b/myisam/ft_nlq_search.c index 9a728a4c211..95ff700f815 100644 --- a/myisam/ft_search.c +++ b/myisam/ft_nlq_search.c @@ -16,26 +16,46 @@ /* Written by Sergei A. Golubchik, who has a shared copyright to this code */ +#define FT_CORE #include "ftdefs.h" -/* queries isam and returns list of documents matched */ +/* search with natural language queries */ -typedef struct st_all_in_one { +typedef struct ft_doc_rec +{ + my_off_t dpos; + double weight; +} FT_DOC; + +struct st_ft_info +{ + struct _ft_vft *please; + MI_INFO *info; + int ndocs; + int curdoc; + FT_DOC doc[1]; +}; + +typedef struct st_all_in_one +{ MI_INFO *info; uint keynr; + CHARSET_INFO *charset; uchar *keybuff; MI_KEYDEF *keyinfo; my_off_t key_root; TREE dtree; } ALL_IN_ONE; -typedef struct st_ft_superdoc { +typedef struct st_ft_superdoc +{ FT_DOC doc; FT_WORD *word_ptr; double tmp_weight; } FT_SUPERDOC; -static int FT_SUPERDOC_cmp(FT_SUPERDOC *p1, FT_SUPERDOC *p2) +static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)), + FT_SUPERDOC *p1, FT_SUPERDOC *p2) { if (p1->doc.dpos < p2->doc.dpos) return -1; @@ -59,6 +79,8 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) #error #endif + DBUG_ENTER("walk_and_match"); + word->weight=LWS_FOR_QUERY; keylen=_ft_make_key(aio->info,aio->keynr,(char*) aio->keybuff,word,0); @@ -75,10 +97,11 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) r=_mi_search(aio->info, aio->keyinfo, aio->keybuff, keylen, SEARCH_FIND | SEARCH_PREFIX, aio->key_root); + aio->info->update|= HA_STATE_AKTIV; /* for _mi_test_if_changed() */ - while(!r) + while (!r) { - if (_mi_compare_text(default_charset_info, + if (_mi_compare_text(aio->charset, aio->info->lastkey,keylen, aio->keybuff,keylen,0)) break; @@ -91,7 +114,7 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) #else #error #endif - if(tmp_weight==0) return doc_cnt; /* stopword, doc_cnt should be 0 */ + if(tmp_weight==0) DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */ #ifdef EVAL_RUN cnt=*(byte *)(aio->info->lastkey+keylen); @@ -100,11 +123,12 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) sdoc.doc.dpos=aio->info->lastpos; /* saving document matched into dtree */ - if(!(selem=tree_insert(&aio->dtree, &sdoc, 0))) return 1; + if (!(selem=tree_insert(&aio->dtree, &sdoc, 0))) + DBUG_RETURN(1); sptr=(FT_SUPERDOC *)ELEMENT_KEY((&aio->dtree), selem); - if(selem->count==1) /* document's first match */ + if (selem->count==1) /* document's first match */ sptr->doc.weight=0; else sptr->doc.weight+=sptr->tmp_weight*sptr->word_ptr->weight; @@ -128,88 +152,101 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) aio->info->lastkey_length, SEARCH_BIGGER, aio->key_root); } - if(doc_cnt) { + if (doc_cnt) + { word->weight*=GWS_IN_USE; - if(word->weight < 0) word->weight=0; - } + if (word->weight < 0) + word->weight=0; - return 0; + } + DBUG_RETURN(0); } + static int walk_and_copy(FT_SUPERDOC *from, uint32 count __attribute__((unused)), FT_DOC **to) { - from->doc.weight+=from->tmp_weight*from->word_ptr->weight; - (*to)->dpos=from->doc.dpos; - (*to)->weight=from->doc.weight; - (*to)++; - return 0; + DBUG_ENTER("walk_and_copy"); + from->doc.weight+=from->tmp_weight*from->word_ptr->weight; + (*to)->dpos=from->doc.dpos; + (*to)->weight=from->doc.weight; + (*to)++; + DBUG_RETURN(0); } + static int FT_DOC_cmp(FT_DOC *a, FT_DOC *b) { - return sgn(b->weight - a->weight); + return sgn(b->weight - a->weight); } -FT_DOCLIST * ft_init_search(void *info, uint keynr, byte *key, - uint key_len, my_bool presort) + +FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query, + uint query_len, my_bool presort) { - TREE *wtree; - ALL_IN_ONE aio; - FT_DOCLIST *dlist; + TREE allocated_wtree, *wtree=&allocated_wtree; + ALL_IN_ONE aio; FT_DOC *dptr; - my_off_t saved_lastpos=((MI_INFO *)info)->lastpos; + FT_INFO *dlist=NULL; + my_off_t saved_lastpos=info->lastpos; + DBUG_ENTER("ft_init_nlq_search"); /* black magic ON */ - if ((int) (keynr = _mi_check_index((MI_INFO *)info,keynr)) < 0) - return NULL; - if (_mi_readinfo((MI_INFO *)info,F_RDLCK,1)) - return NULL; + if ((int) (keynr = _mi_check_index(info,keynr)) < 0) + DBUG_RETURN(NULL); + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(NULL); /* black magic OFF */ - dlist=NULL; - aio.info=(MI_INFO *)info; + aio.info=info; aio.keynr=keynr; - aio.keybuff=aio.info->lastkey+aio.info->s->base.max_key_length; - aio.keyinfo=aio.info->s->keyinfo+keynr; - aio.key_root=aio.info->s->state.key_root[keynr]; + aio.keyinfo=info->s->keyinfo+keynr; + aio.charset=aio.keyinfo->seg->charset; + aio.keybuff=info->lastkey+info->s->base.max_key_length; + aio.key_root=info->s->state.key_root[keynr]; + + bzero(&allocated_wtree,sizeof(allocated_wtree)); - if (!(wtree=ft_parse(NULL,key,key_len))) return NULL; + init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0, + NULL, NULL); - init_tree(&aio.dtree,0,sizeof(FT_SUPERDOC),(qsort_cmp)&FT_SUPERDOC_cmp,0, - NULL); + ft_parse_init(&allocated_wtree, aio.charset); + if (ft_parse(&allocated_wtree,query,query_len)) + goto err; if (tree_walk(wtree, (tree_walk_action)&walk_and_match, &aio, left_root_right)) - goto err; + goto err2; - dlist=(FT_DOCLIST *) my_malloc(sizeof(FT_DOCLIST)+sizeof(FT_DOC)* - (aio.dtree.elements_in_tree-1),MYF(0)); - if (!dlist) - goto err; + dlist=(FT_INFO *)my_malloc(sizeof(FT_INFO)+ + sizeof(FT_DOC)*(aio.dtree.elements_in_tree-1), + MYF(0)); + if(!dlist) + goto err2; + dlist->please= (struct _ft_vft *) & _ft_vft_nlq; dlist->ndocs=aio.dtree.elements_in_tree; dlist->curdoc=-1; dlist->info=aio.info; dptr=dlist->doc; - tree_walk(&aio.dtree, (tree_walk_action)&walk_and_copy, &dptr, - left_root_right); + tree_walk(&aio.dtree, (tree_walk_action) &walk_and_copy, + &dptr, left_root_right); if (presort) - { qsort(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort_cmp)&FT_DOC_cmp); - } -err: - delete_tree(&aio.dtree); +err2: delete_tree(wtree); - my_free((char*) wtree,MYF(0)); - ((MI_INFO *)info)->lastpos=saved_lastpos; - return dlist; + delete_tree(&aio.dtree); + +err: + info->lastpos=saved_lastpos; + DBUG_RETURN(dlist); } -int ft_read_next(FT_DOCLIST *handler, char *record) + +int ft_nlq_read_next(FT_INFO *handler, char *record) { MI_INFO *info= (MI_INFO *) handler->info; @@ -229,3 +266,49 @@ int ft_read_next(FT_DOCLIST *handler, char *record) } return my_errno; } + + +float ft_nlq_find_relevance(FT_INFO *handler, + byte *record __attribute__((unused)), + uint length __attribute__((unused))) +{ + int a,b,c; + FT_DOC *docs=handler->doc; + my_off_t docid=handler->info->lastpos; + + if (docid == HA_POS_ERROR) + return -5.0; + + /* Assuming docs[] is sorted by dpos... */ + + for (a=0, b=handler->ndocs, c=(a+b)/2; b-a>1; c=(a+b)/2) + { + if (docs[c].dpos > docid) + b=c; + else + a=c; + } + if (docs[a].dpos == docid) + return (float) docs[a].weight; + else + return 0.0; +} + + +void ft_nlq_close_search(FT_INFO *handler) +{ + my_free((gptr)handler,MYF(0)); +} + + +float ft_nlq_get_relevance(FT_INFO *handler) +{ + return (float) handler->doc[handler->curdoc].weight; +} + + +void ft_nlq_reinit_search(FT_INFO *handler) +{ + handler->curdoc=-1; +} + diff --git a/myisam/ft_parser.c b/myisam/ft_parser.c index 7ea2e240c36..c25ed6022a0 100644 --- a/myisam/ft_parser.c +++ b/myisam/ft_parser.c @@ -33,16 +33,12 @@ typedef struct st_ft_docstat { double max, nsum, nsum2; #endif /* EVAL_RUN */ - MI_INFO *info; - uint keynr; - byte *keybuf; } FT_DOCSTAT; -static int FT_WORD_cmp(FT_WORD *w1, FT_WORD *w2) +static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2) { - return _mi_compare_text(default_charset_info, - (uchar*) w1->pos,w1->len, - (uchar*) w2->pos, w2->len,0); + return _mi_compare_text(cs, (uchar*) w1->pos, w1->len, + (uchar*) w2->pos, w2->len, 0); } static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) @@ -63,17 +59,15 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) /* transforms tree of words into the array, applying normalization */ -FT_WORD * ft_linearize(MI_INFO *info, uint keynr, byte *keybuf, TREE *wtree) +FT_WORD * ft_linearize(TREE *wtree) { FT_WORD *wlist,*p; FT_DOCSTAT docstat; + DBUG_ENTER("ft_linearize"); if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)* (1+wtree->elements_in_tree),MYF(0)))) { - docstat.info=info; - docstat.keynr=keynr; - docstat.keybuf=keybuf; docstat.list=wlist; docstat.uniq=wtree->elements_in_tree; #ifdef EVAL_RUN @@ -83,13 +77,12 @@ FT_WORD * ft_linearize(MI_INFO *info, uint keynr, byte *keybuf, TREE *wtree) tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right); } delete_tree(wtree); - my_free((char*) wtree,MYF(0)); if (!wlist) - return NULL; + DBUG_RETURN(NULL); docstat.list->pos=NULL; - for(p=wlist;p->pos;p++) + for (p=wlist;p->pos;p++) { p->weight=PRENORM_IN_USE; #ifdef EVAL_RUN @@ -104,48 +97,146 @@ FT_WORD * ft_linearize(MI_INFO *info, uint keynr, byte *keybuf, TREE *wtree) #endif #endif /* EVAL_RUN */ - for(p=wlist;p->pos;p++) + for (p=wlist;p->pos;p++) { p->weight/=NORM_IN_USE; } - return wlist; + DBUG_RETURN(wlist); } +#define true_word_char(X) (isalnum(X) || (X)=='_') #ifdef HYPHEN_IS_DELIM -#define word_char(X) (isalnum(X) || (X)=='_' || (X)=='\'') +#define misc_word_char(X) ((X)=='\'') #else -#define word_char(X) (isalnum(X) || (X)=='_' || (X)=='\'' || (X)=='-') +#define misc_word_char(X) ((X)=='\'' || (X)=='-') #endif +#define word_char(X) (true_word_char(X) || misc_word_char(X)) -/* this is rather dumb first version of the parser */ -TREE * ft_parse(TREE *wtree, byte *doc, int doclen) + +/* returns: + * 0 - eof + * 1 - word found + * 2 - left bracket + * 3 - right bracket + */ +byte ft_get_word(byte **start, byte *end, FT_WORD *word, FTB_PARAM *param) { - byte *end=doc+doclen; - FT_WORD w; + byte *doc=*start; + int mwc; + + param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0); + param->plusminus=param->pmsign=0; - if (!wtree) + while (doc<end) { - if (!(wtree=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) return NULL; - init_tree(wtree,0,sizeof(FT_WORD),(qsort_cmp)&FT_WORD_cmp,0,NULL); + for (;doc<end;doc++) + { + if (true_word_char(*doc)) break; + if (*doc == FTB_RQUOT && param->quot) { + param->quot=doc; + *start=doc+1; + return 3; /* FTB_RBR */ + } + if ((*doc == FTB_LBR || *doc == FTB_RBR || *doc == FTB_LQUOT) + && !param->quot) + { + /* param->prev=' '; */ + *start=doc+1; + if (*doc == FTB_LQUOT) param->quot=*start; + return (*doc == FTB_RBR)+2; + } + if (param->prev == ' ' && !param->quot) + { + if (*doc == FTB_YES ) { param->yesno=+1; continue; } else + if (*doc == FTB_EGAL) { param->yesno= 0; continue; } else + if (*doc == FTB_NO ) { param->yesno=-1; continue; } else + if (*doc == FTB_INC ) { param->plusminus++; continue; } else + if (*doc == FTB_DEC ) { param->plusminus--; continue; } else + if (*doc == FTB_NEG ) { param->pmsign=!param->pmsign; continue; } + } + param->prev=*doc; + param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0); + param->plusminus=param->pmsign=0; + } + + mwc=0; + for (word->pos=doc; doc<end; doc++) + if (true_word_char(*doc)) + mwc=0; + else if (!misc_word_char(*doc) || mwc++) + break; + + param->prev='A'; /* be sure *prev is true_word_char */ + word->len= (uint)(doc-word->pos) - mwc; + if ((param->trunc=(doc<end && *doc == FTB_TRUNC))) + doc++; + + if (((word->len >= ft_min_word_len && !is_stopword(word->pos, word->len)) + || param->trunc) && word->len < ft_max_word_len) + { + *start=doc; + return 1; + } } + return 0; +} + +byte ft_simple_get_word(byte **start, byte *end, FT_WORD *word) +{ + byte *doc=*start; + int mwc; + DBUG_ENTER("ft_simple_get_word"); - w.weight=0; while (doc<end) { for (;doc<end;doc++) - if (word_char(*doc)) break; - for (w.pos=doc; doc<end; doc++) - if (!word_char(*doc)) break; - if ((w.len= (uint) (doc-w.pos)) < MIN_WORD_LEN) continue; - if (w.len >= HA_FT_MAXLEN) continue; - if (is_stopword(w.pos, w.len)) continue; - if (!tree_insert(wtree, &w, 0)) { - delete_tree(wtree); - my_free((char*) wtree,MYF(0)); - return NULL; + if (true_word_char(*doc)) break; + } + + mwc=0; + for(word->pos=doc; doc<end; doc++) + if (true_word_char(*doc)) + mwc=0; + else if (!misc_word_char(*doc) || mwc++) + break; + + word->len= (uint)(doc-word->pos) - mwc; + + if (word->len >= ft_min_word_len && word->len < ft_max_word_len && + !is_stopword(word->pos, word->len)) + { + *start=doc; + DBUG_RETURN(1); } } - return wtree; + DBUG_RETURN(0); } + +void ft_parse_init(TREE *wtree, CHARSET_INFO *cs) +{ + DBUG_ENTER("ft_parse_init"); + if (!is_tree_inited(wtree)) + init_tree(wtree,0,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL, cs); + DBUG_VOID_RETURN; +} + +int ft_parse(TREE *wtree, byte *doc, int doclen) +{ + byte *end=doc+doclen; + FT_WORD w; + DBUG_ENTER("ft_parse"); + + while (ft_simple_get_word(&doc,end,&w)) + { + if (!tree_insert(wtree, &w, 0)) + goto err; + } + DBUG_RETURN(0); + +err: + delete_tree(wtree); + DBUG_RETURN(1); +} + diff --git a/myisam/ft_static.c b/myisam/ft_static.c index 00d9d4ed19a..7f78a11bb2f 100644 --- a/myisam/ft_static.c +++ b/myisam/ft_static.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -18,6 +18,11 @@ #include "ftdefs.h" +ulong ft_min_word_len=4; +ulong ft_max_word_len=HA_FT_MAXLEN; +ulong ft_max_word_len_for_sort=20; +const char *ft_boolean_syntax="+ -><()~*:\"\"&|"; + const MI_KEYSEG ft_keysegs[FT_SEGS]={ { HA_KEYTYPE_VARTEXT, /* type */ @@ -39,10 +44,30 @@ const MI_KEYSEG ft_keysegs[FT_SEGS]={ }, #endif /* EVAL_RUN */ { - HA_FT_WTYPE, 7, 0, 0, 0, 0, HA_FT_WLEN, 0, 0, NULL + HA_FT_WTYPE, 7, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 0, 0, NULL } }; +const struct _ft_vft _ft_vft_nlq = { + ft_nlq_read_next, ft_nlq_find_relevance, ft_nlq_close_search, + ft_nlq_get_relevance, ft_nlq_reinit_search +}; +const struct _ft_vft _ft_vft_boolean = { + ft_boolean_read_next, ft_boolean_find_relevance, ft_boolean_close_search, + ft_boolean_get_relevance, ft_boolean_reinit_search +}; + +FT_INFO *(*_ft_init_vft[2])(MI_INFO *, uint, byte *, uint, my_bool) = +{ ft_init_nlq_search, ft_init_boolean_search }; + +FT_INFO *ft_init_search(uint mode, void *info, uint keynr, + byte *query, uint query_len, my_bool presort) +{ + return (*_ft_init_vft[mode])((MI_INFO *)info, keynr, + query, query_len, presort); +} + +const char *ft_stopword_file = 0; const char *ft_precompiled_stopwords[] = { #ifdef COMPILE_STOPWORDS_IN @@ -52,7 +77,6 @@ const char *ft_precompiled_stopwords[] = { it was slightly modified to my taste, though */ - "a", "a's", "able", "about", @@ -106,7 +130,6 @@ const char *ft_precompiled_stopwords[] = { "available", "away", "awfully", - "b", "be", "became", "because", @@ -130,7 +153,6 @@ const char *ft_precompiled_stopwords[] = { "brief", "but", "by", - "c", "c'mon", "c's", "came", @@ -160,7 +182,6 @@ const char *ft_precompiled_stopwords[] = { "couldn't", "course", "currently", - "d", "definitely", "described", "despite", @@ -176,7 +197,6 @@ const char *ft_precompiled_stopwords[] = { "down", "downwards", "during", - "e", "each", "edu", "eg", @@ -200,7 +220,6 @@ const char *ft_precompiled_stopwords[] = { "exactly", "example", "except", - "f", "far", "few", "fifth", @@ -217,7 +236,6 @@ const char *ft_precompiled_stopwords[] = { "from", "further", "furthermore", - "g", "get", "gets", "getting", @@ -230,7 +248,6 @@ const char *ft_precompiled_stopwords[] = { "got", "gotten", "greetings", - "h", "had", "hadn't", "happens", @@ -263,7 +280,6 @@ const char *ft_precompiled_stopwords[] = { "how", "howbeit", "however", - "i", "i'd", "i'll", "i'm", @@ -292,16 +308,13 @@ const char *ft_precompiled_stopwords[] = { "it's", "its", "itself", - "j", "just", - "k", "keep", "keeps", "kept", "know", "knows", "known", - "l", "last", "lately", "later", @@ -320,7 +333,6 @@ const char *ft_precompiled_stopwords[] = { "looking", "looks", "ltd", - "m", "mainly", "many", "may", @@ -338,7 +350,6 @@ const char *ft_precompiled_stopwords[] = { "must", "my", "myself", - "n", "name", "namely", "nd", @@ -365,7 +376,6 @@ const char *ft_precompiled_stopwords[] = { "novel", "now", "nowhere", - "o", "obviously", "of", "off", @@ -393,7 +403,6 @@ const char *ft_precompiled_stopwords[] = { "over", "overall", "own", - "p", "particular", "particularly", "per", @@ -405,11 +414,9 @@ const char *ft_precompiled_stopwords[] = { "presumably", "probably", "provides", - "q", "que", "quite", "qv", - "r", "rather", "rd", "re", @@ -421,7 +428,6 @@ const char *ft_precompiled_stopwords[] = { "relatively", "respectively", "right", - "s", "said", "same", "saw", @@ -471,7 +477,6 @@ const char *ft_precompiled_stopwords[] = { "such", "sup", "sure", - "t", "t's", "take", "taken", @@ -531,7 +536,6 @@ const char *ft_precompiled_stopwords[] = { "trying", "twice", "two", - "u", "un", "under", "unfortunately", @@ -548,14 +552,12 @@ const char *ft_precompiled_stopwords[] = { "uses", "using", "usually", - "v", "value", "various", "very", "via", "viz", "vs", - "w", "want", "wants", "was", @@ -607,8 +609,6 @@ const char *ft_precompiled_stopwords[] = { "would", "would", "wouldn't", - "x", - "y", "yes", "yet", "you", @@ -620,7 +620,6 @@ const char *ft_precompiled_stopwords[] = { "yours", "yourself", "yourselves", - "z", "zero", #endif diff --git a/myisam/ft_stem.c b/myisam/ft_stem.c index bdfc73b774b..846d5d2247f 100644 --- a/myisam/ft_stem.c +++ b/myisam/ft_stem.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -17,4 +17,3 @@ /* Written by Sergei A. Golubchik, who has a shared copyright to this code */ /* mulitingual stem */ - diff --git a/myisam/ft_stopwords.c b/myisam/ft_stopwords.c index d796b87ed71..298df9a54cf 100644 --- a/myisam/ft_stopwords.c +++ b/myisam/ft_stopwords.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -25,35 +25,83 @@ typedef struct st_ft_stopwords { static TREE *stopwords3=NULL; -static int FT_STOPWORD_cmp(FT_STOPWORD *w1, FT_STOPWORD *w2) +static int FT_STOPWORD_cmp(void* cmp_arg __attribute__((unused)), + FT_STOPWORD *w1, FT_STOPWORD *w2) { return _mi_compare_text(default_charset_info, (uchar *)w1->pos,w1->len, (uchar *)w2->pos,w2->len,0); } -int ft_init_stopwords(const char **sws) +static void FT_STOPWORD_free(FT_STOPWORD *w, TREE_FREE action, + void *arg __attribute__((unused))) { - FT_STOPWORD sw; + if (action == free_free) + my_free((gptr) w->pos, MYF(0)); +} +static int ft_add_stopword(const char *w) +{ + FT_STOPWORD sw; + return !w || + (((sw.len= (uint) strlen(sw.pos=w)) >= ft_min_word_len) && + (tree_insert(stopwords3, &sw, 0)==NULL)); +} - if(!stopwords3) +int ft_init_stopwords() +{ + if (!stopwords3) { - if(!(stopwords3=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) return -1; - init_tree(stopwords3,0,sizeof(FT_STOPWORD),(qsort_cmp)&FT_STOPWORD_cmp,0, - NULL); + if (!(stopwords3=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) + return -1; + init_tree(stopwords3,0,0,sizeof(FT_STOPWORD),(qsort_cmp2)&FT_STOPWORD_cmp, + 0, + (ft_stopword_file ? (tree_element_free)&FT_STOPWORD_free : 0), + NULL); } - if(!sws) return 0; + if (ft_stopword_file) + { + File fd; + uint len; + byte *buffer, *start, *end; + FT_WORD w; + int error=-1; + + if (!*ft_stopword_file) + return 0; - for(;*sws;sws++) + if ((fd=my_open(ft_stopword_file, O_RDONLY, MYF(MY_WME))) == -1) + return -1; + len=(uint)my_seek(fd, 0L, MY_SEEK_END, MYF(0)); + my_seek(fd, 0L, MY_SEEK_SET, MYF(0)); + if (!(start=buffer=my_malloc(len+1, MYF(MY_WME)))) + goto err0; + len=my_read(fd, buffer, len, MYF(MY_WME)); + end=start+len; + while (ft_simple_get_word(&start, end, &w)) + { + if (ft_add_stopword(my_strdup_with_length(w.pos, w.len, MYF(0)))) + goto err1; + } + error=0; +err1: + my_free(buffer, MYF(0)); +err0: + my_close(fd, MYF(MY_WME)); + return error; + } + else { - if( (sw.len= (uint) strlen(sw.pos=*sws)) < MIN_WORD_LEN) continue; - if(!tree_insert(stopwords3, &sw, 0)) + /* compatibility mode: to be removed */ + char **sws=(char **)ft_precompiled_stopwords; + + for (;*sws;sws++) { - delete_tree(stopwords3); /* purecov: inspected */ - return -1; /* purecov: inspected */ + if (ft_add_stopword(*sws)) + return -1; } + ft_stopword_file="(built-in)"; /* for SHOW VARIABLES */ } return 0; } @@ -71,7 +119,7 @@ void ft_free_stopwords() { if (stopwords3) { - delete_tree(stopwords3); /* purecov: inspected */ + delete_tree(stopwords3); /* purecov: inspected */ my_free((char*) stopwords3,MYF(0)); stopwords3=0; } diff --git a/myisam/ft_test1.c b/myisam/ft_test1.c index 5093b591fb2..cb0b6054f0a 100644 --- a/myisam/ft_test1.c +++ b/myisam/ft_test1.c @@ -14,13 +14,14 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ +/* Written by Sergei A. Golubchik, who has a shared copyright to this code + added support for long options (my_getopt) 22.5.2002 by Jani Tolonen */ #include "ftdefs.h" #include "ft_test1.h" -#include <getopt.h> +#include <my_getopt.h> -static int key_field=FIELD_VARCHAR,extra_field=FIELD_SKIPP_ENDSPACE; +static int key_field=FIELD_VARCHAR,extra_field=FIELD_SKIP_ENDSPACE; static uint key_length=200,extra_length=50; static int key_type=HA_KEYTYPE_TEXT; static int verbose=0,silent=0,skip_update=0, @@ -33,8 +34,26 @@ static char record[MAX_REC_LENGTH],read_record[MAX_REC_LENGTH]; static int run_test(const char *filename); static void get_options(int argc, char *argv[]); static void create_record(char *, int); +static void usage(); -int main(int argc,char *argv[]) +static struct my_option my_long_options[] = +{ + {"", 'v', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", '?', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'h', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'V', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'v', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 's', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'N', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'S', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'K', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'F', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", 'U', "", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"", '#', "", 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +int main(int argc, char *argv[]) { MY_INIT(argv[0]); @@ -137,7 +156,7 @@ static int run_test(const char *filename) printf("- Reading rows with key\n"); for (i=0 ; i < NQUERIES ; i++) { FT_DOCLIST *result; - result=ft_init_search(file,0,(char*) query[i],strlen(query[i]),1); + result=ft_nlq_init_search(file,0,(char*) query[i],strlen(query[i]),1); if(!result) { printf("Query %d: `%s' failed with errno %3d\n",i,query[i],my_errno); continue; @@ -145,7 +164,7 @@ static int run_test(const char *filename) printf("Query %d: `%s'. Found: %d. Top five documents:\n", i,query[i],result->ndocs); for(j=0;j<5;j++) { double w; int err; - err=ft_read_next(result, read_record); + err=ft_nlq_read_next(result, read_record); if(err==HA_ERR_END_OF_FILE) { printf("No more matches!\n"); break; @@ -153,7 +172,7 @@ static int run_test(const char *filename) printf("ft_read_next %d failed with errno %3d\n",j,my_errno); break; } - w=ft_get_relevance(result); + w=ft_nlq_get_relevance(result); if(key_field == FIELD_VARCHAR) { uint l; char *p; @@ -164,7 +183,7 @@ static int run_test(const char *filename) printf("%10.7f: %.*s\n",w,recinfo[1].length, recinfo[0].length+read_record); } - ft_close_search(result); + ft_nlq_close_search(result); } if (mi_close(file)) goto err; @@ -173,7 +192,7 @@ static int run_test(const char *filename) return (0); err: printf("got error: %3d when using myisam-database\n",my_errno); - return 1; /* skipp warning */ + return 1; /* skip warning */ } static char blob_key[MAX_REC_LENGTH]; @@ -186,7 +205,7 @@ void create_record(char *pos, int n) { uint tmp; char *ptr; - strncpy(blob_key,data[n].f0,keyinfo[0].seg[0].length); + strnmov(blob_key,data[n].f0,keyinfo[0].seg[0].length); tmp=strlen(blob_key); int4store(pos,tmp); ptr=blob_key; @@ -196,21 +215,21 @@ void create_record(char *pos, int n) else if (recinfo[0].type == FIELD_VARCHAR) { uint tmp; - strncpy(pos+2,data[n].f0,keyinfo[0].seg[0].length); + strnmov(pos+2,data[n].f0,keyinfo[0].seg[0].length); tmp=strlen(pos+2); int2store(pos,tmp); pos+=recinfo[0].length; } else { - strncpy(pos,data[n].f0,keyinfo[0].seg[0].length); + strnmov(pos,data[n].f0,keyinfo[0].seg[0].length); pos+=recinfo[0].length; } if (recinfo[1].type == FIELD_BLOB) { uint tmp; char *ptr; - strncpy(blob_key,data[n].f2,keyinfo[0].seg[0].length); + strnmov(blob_key,data[n].f2,keyinfo[0].seg[0].length); tmp=strlen(blob_key); int4store(pos,tmp); ptr=blob_key; @@ -220,46 +239,59 @@ void create_record(char *pos, int n) else if (recinfo[1].type == FIELD_VARCHAR) { uint tmp; - strncpy(pos+2,data[n].f2,keyinfo[0].seg[0].length); + strnmov(pos+2,data[n].f2,keyinfo[0].seg[0].length); tmp=strlen(pos+2); int2store(pos,tmp); pos+=recinfo[1].length; } else { - strncpy(pos,data[n].f2,keyinfo[0].seg[0].length); + strnmov(pos,data[n].f2,keyinfo[0].seg[0].length); pos+=recinfo[1].length; } } + +static my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case 'v': verbose=1; break; + case 's': silent=1; break; + case 'F': no_fulltext=1; no_search=1; + case 'U': skip_update=1; break; + case 'K': no_keys=no_search=1; break; + case 'N': no_search=1; break; + case 'S': no_stopwords=1; break; + case '#': + DEBUGGER_ON; + DBUG_PUSH (argument); + break; + case 'V': + case '?': + case 'h': + usage(); + exit(1); + } + return 0; +} + /* Read options */ static void get_options(int argc,char *argv[]) { - int c; - const char *options="hVvsNSKFU#:"; + int ho_error; - while ((c=getopt(argc,argv,options)) != -1) - { - switch(c) { - case 'v': verbose=1; break; - case 's': silent=1; break; - case 'F': no_fulltext=1; no_search=1; - case 'U': skip_update=1; break; - case 'K': no_keys=no_search=1; break; - case 'N': no_search=1; break; - case 'S': no_stopwords=1; break; - case '#': - DEBUGGER_ON; - DBUG_PUSH (optarg); - break; - case 'V': - case '?': - case 'h': - default: - printf("%s -[%s]\n", argv[0], options); - exit(0); - } - } + if ((ho_error=handle_options(&argc, &argv, my_long_options, get_one_option))) + exit(ho_error); return; } /* get options */ + + +static void usage() +{ + printf("%s [options]\n", my_progname); + my_print_help(my_long_options); + my_print_variables(my_long_options); +} diff --git a/myisam/ft_test1.h b/myisam/ft_test1.h index 17b0cae66b7..e360244057b 100644 --- a/myisam/ft_test1.h +++ b/myisam/ft_test1.h @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -419,4 +419,3 @@ const char *query[NQUERIES]={ "against about after more right the with/without", /* stopwords test */ "mysql license and copyright" }; - diff --git a/myisam/ft_update.c b/myisam/ft_update.c index 753c4dc4029..a68cc2a4cf4 100644 --- a/myisam/ft_update.c +++ b/myisam/ft_update.c @@ -19,6 +19,7 @@ /* functions to work with full-text indices */ #include "ftdefs.h" +#include <math.h> /************************************************************** This is to make ft-code to ignore keyseg.length at all * @@ -27,46 +28,92 @@ #define set_if_smaller(A,B) /* no op */ /**************************************************************/ +void _mi_ft_segiterator_init(MI_INFO *info, uint keynr, const byte *record, + FT_SEG_ITERATOR *ftsi) +{ + ftsi->num=info->s->keyinfo[keynr].keysegs-FT_SEGS; + ftsi->seg=info->s->keyinfo[keynr].seg; + ftsi->rec=record; +} + +void _mi_ft_segiterator_dummy_init(const byte *record, uint len, + FT_SEG_ITERATOR *ftsi) +{ + ftsi->num=1; + ftsi->seg=0; + ftsi->pos=record; + ftsi->len=len; +} + +/* + This function breaks convention "return 0 in success" + but it's easier to use like this + + while(_mi_ft_segiterator()) + + so "1" means "OK", "0" means "EOF" +*/ -/* parses a document i.e. calls _mi_ft_parse for every keyseg */ -static FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, byte *keybuf, - const byte *record) +uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi) { - TREE *parsed=NULL; - MI_KEYSEG *keyseg; - byte *pos; - uint i; + if (!ftsi->num) return 0; else ftsi->num--; + if (!ftsi->seg) return 1; else ftsi->seg--; - keyseg=info->s->keyinfo[keynr].seg; - for (i=info->s->keyinfo[keynr].keysegs-FT_SEGS ; i-- ; ) + if (ftsi->seg->null_bit && + (ftsi->rec[ftsi->seg->null_pos] & ftsi->seg->null_bit)) { - uint len; + ftsi->pos=0; + return 1; + } + ftsi->pos= ftsi->rec+ftsi->seg->start; + if (ftsi->seg->flag & HA_VAR_LENGTH) + { + ftsi->len=uint2korr(ftsi->pos); + ftsi->pos+=2; /* Skip VARCHAR length */ + set_if_smaller(ftsi->len,ftsi->seg->length); + return 1; + } + if (ftsi->seg->flag & HA_BLOB_PART) + { + ftsi->len=_mi_calc_blob_length(ftsi->seg->bit_start,ftsi->pos); + memcpy_fixed((char*) &ftsi->pos, ftsi->pos+ftsi->seg->bit_start, + sizeof(char*)); + set_if_smaller(ftsi->len,ftsi->seg->length); + return 1; + } + ftsi->len=ftsi->seg->length; + return 1; +} - keyseg--; - if (keyseg->null_bit && (record[keyseg->null_pos] & keyseg->null_bit)) - continue; /* NULL field */ - pos= (byte *)record+keyseg->start; - if (keyseg->flag & HA_VAR_LENGTH) - { - len=uint2korr(pos); - pos+=2; /* Skip VARCHAR length */ - set_if_smaller(len,keyseg->length); - } - else if (keyseg->flag & HA_BLOB_PART) - { - len=_mi_calc_blob_length(keyseg->bit_start,pos); - memcpy_fixed(&pos,pos+keyseg->bit_start,sizeof(char*)); - set_if_smaller(len,keyseg->length); - } - else - len=keyseg->length; - if (!(parsed=ft_parse(parsed, pos, len))) - return NULL; + +/* parses a document i.e. calls ft_parse for every keyseg */ + +uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record) +{ + FT_SEG_ITERATOR ftsi; + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + + ft_parse_init(parsed, info->s->keyinfo[keynr].seg->charset); + while (_mi_ft_segiterator(&ftsi)) + { + if (ftsi.pos) + if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len)) + return 1; } - /* Handle the case where all columns are NULL */ - if (!parsed && !(parsed=ft_parse(0, (byte*) "", 0))) + return 0; +} + +FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, + byte *keybuf __attribute__((unused)), + const byte *record) +{ + TREE ptree; + + bzero((char*) &ptree, sizeof(ptree)); + if (_mi_ft_parse(&ptree, info, keynr, record)) return NULL; - return ft_linearize(info, keynr, keybuf, parsed); + + return ft_linearize(/*info, keynr, keybuf, */ &ptree); } static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf, @@ -74,88 +121,110 @@ static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf, { uint key_length; - while(wlist->pos) + for (; wlist->pos; wlist++) { key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); if (_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)) return 1; - wlist++; } return 0; } -static int _mi_ft_erase(MI_INFO *info, uint keynr, byte *keybuf, FT_WORD *wlist, my_off_t filepos) +static int _mi_ft_erase(MI_INFO *info, uint keynr, byte *keybuf, + FT_WORD *wlist, my_off_t filepos) { uint key_length, err=0; - while(wlist->pos) + for (; wlist->pos; wlist++) { key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); if (_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)) err=1; - wlist++; } return err; } -/* compares an appropriate parts of two WORD_KEY keys directly out of records */ -/* returns 1 if they are different */ +/* + Compares an appropriate parts of two WORD_KEY keys directly out of records + returns 1 if they are different +*/ #define THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT 1 #define GEE_THEY_ARE_ABSOLUTELY_IDENTICAL 0 int _mi_ft_cmp(MI_INFO *info, uint keynr, const byte *rec1, const byte *rec2) { - MI_KEYSEG *keyseg; - byte *pos1, *pos2; - uint i; + FT_SEG_ITERATOR ftsi1, ftsi2; + CHARSET_INFO *cs=info->s->keyinfo[keynr].seg->charset; + _mi_ft_segiterator_init(info, keynr, rec1, &ftsi1); + _mi_ft_segiterator_init(info, keynr, rec2, &ftsi2); - i=info->s->keyinfo[keynr].keysegs-FT_SEGS; - keyseg=info->s->keyinfo[keynr].seg; - while(i--) + while (_mi_ft_segiterator(&ftsi1) && _mi_ft_segiterator(&ftsi2)) { - uint len1, len2; - LINT_INIT(len1); LINT_INIT(len2); - keyseg--; - if (keyseg->null_bit) - { - if ( (rec1[keyseg->null_pos] ^ rec2[keyseg->null_pos]) - & keyseg->null_bit ) - return THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT; - if (rec1[keyseg->null_pos] & keyseg->null_bit ) - continue; /* NULL field */ - } - pos1= (byte *)rec1+keyseg->start; - pos2= (byte *)rec2+keyseg->start; - if (keyseg->flag & HA_VAR_LENGTH) - { - len1=uint2korr(pos1); - pos1+=2; /* Skip VARCHAR length */ - set_if_smaller(len1,keyseg->length); - len2=uint2korr(pos2); - pos2+=2; /* Skip VARCHAR length */ - set_if_smaller(len2,keyseg->length); - } - else if (keyseg->flag & HA_BLOB_PART) + if ((ftsi1.pos != ftsi2.pos) && + (!ftsi1.pos || !ftsi2.pos || + _mi_compare_text(cs, (uchar*) ftsi1.pos,ftsi1.len, + (uchar*) ftsi2.pos,ftsi2.len,0))) + return THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT; + } + return GEE_THEY_ARE_ABSOLUTELY_IDENTICAL; +} + + +/* update a document entry */ + +int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, + const byte *oldrec, const byte *newrec, my_off_t pos) +{ + int error= -1; + FT_WORD *oldlist,*newlist, *old_word, *new_word; + CHARSET_INFO *cs=info->s->keyinfo[keynr].seg->charset; + uint key_length; + int cmp, cmp2; + + if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, keybuf, oldrec))) + goto err0; + if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, keybuf, newrec))) + goto err1; + + error=0; + while(old_word->pos && new_word->pos) + { + cmp=_mi_compare_text(cs, (uchar*) old_word->pos,old_word->len, + (uchar*) new_word->pos,new_word->len,0); + cmp2= cmp ? 0 : (fabs(old_word->weight - new_word->weight) > 1.e-5); + + if (cmp < 0 || cmp2) { - len1=_mi_calc_blob_length(keyseg->bit_start,pos1); - memcpy_fixed(&pos1,pos1+keyseg->bit_start,sizeof(char*)); - set_if_smaller(len1,keyseg->length); - len2=_mi_calc_blob_length(keyseg->bit_start,pos2); - memcpy_fixed(&pos2,pos2+keyseg->bit_start,sizeof(char*)); - set_if_smaller(len2,keyseg->length); + key_length=_ft_make_key(info,keynr,keybuf,old_word,pos); + if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length))) + goto err2; } - else /* fixed length key */ + if (cmp > 0 || cmp2) { - len1=len2=keyseg->length; + key_length=_ft_make_key(info,keynr,keybuf,new_word,pos); + if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length))) + goto err2; } - if ((len1 != len2) || memcmp(pos1, pos2, len1)) - return THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT; - } - return GEE_THEY_ARE_ABSOLUTELY_IDENTICAL; + if (cmp<=0) old_word++; + if (cmp>=0) new_word++; + } + if (old_word->pos) + error=_mi_ft_erase(info,keynr,keybuf,old_word,pos); + else if (new_word->pos) + error=_mi_ft_store(info,keynr,keybuf,new_word,pos); + +err2: + my_free((char*) newlist,MYF(0)); +err1: + my_free((char*) oldlist,MYF(0)); +err0: + return error; } + /* adds a document to the collection */ + int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, my_off_t pos) { @@ -170,7 +239,9 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, return error; } + /* removes a document from the collection */ + int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, my_off_t pos) { diff --git a/myisam/ftdefs.h b/myisam/ftdefs.h index d9b4ff6b44d..62fa4362e19 100644 --- a/myisam/ftdefs.h +++ b/myisam/ftdefs.h @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -22,15 +22,18 @@ #include <m_ctype.h> #include <my_tree.h> -#define MIN_WORD_LEN 4 - #define HYPHEN_IS_DELIM #define HYPHEN_IS_CONCAT /* not used for now */ #define COMPILE_STOPWORDS_IN -/* Most of the formulae were shamelessly stolen from SMART distribution - ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z +/* Interested readers may consult SMART + (ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z) + for an excellent implementation of vector space model we use. + It also demonstrate the usage of different weghting techniques. + This code, though, is completely original and is not based on the + SMART code but was in some cases inspired by it. + NORM_PIVOT was taken from the article A.Singhal, C.Buckley, M.Mitra, "Pivoted Document Length Normalization", ACM SIGIR'96, 21-29, 1996 @@ -75,13 +78,26 @@ extern ulong collstat; /* Mysterious, but w/o (double) GWS_IDF performs better :-o */ #define GWS_IDF log(aio->info->state->records/doc_cnt) #define GWS_IDF1 log((double)aio->info->state->records/doc_cnt) -#define GWS_PROB log(((double)(aio->info->state->records-doc_cnt))/doc_cnt) +#define GWS_PROB ((aio->info->state->records > doc_cnt) ? log(((double)(aio->info->state->records-doc_cnt))/doc_cnt) : 0 ) #define GWS_FREQ (1.0/doc_cnt) #define GWS_SQUARED pow(log((double)aio->info->state->records/doc_cnt),2) #define GWS_CUBIC pow(log((double)aio->info->state->records/doc_cnt),3) #define GWS_ENTROPY (1-(suml/sum-log(sum))/log(aio->info->state->records)) /*=================================================================*/ +/* Boolean search operators */ +#define FTB_YES (ft_boolean_syntax[0]) +#define FTB_EGAL (ft_boolean_syntax[1]) +#define FTB_NO (ft_boolean_syntax[2]) +#define FTB_INC (ft_boolean_syntax[3]) +#define FTB_DEC (ft_boolean_syntax[4]) +#define FTB_LBR (ft_boolean_syntax[5]) +#define FTB_RBR (ft_boolean_syntax[6]) +#define FTB_NEG (ft_boolean_syntax[7]) +#define FTB_TRUNC (ft_boolean_syntax[8]) +#define FTB_LQUOT (ft_boolean_syntax[10]) +#define FTB_RQUOT (ft_boolean_syntax[11]) + typedef struct st_ft_word { byte * pos; uint len; @@ -91,9 +107,52 @@ typedef struct st_ft_word { #endif /* EVAL_RUN */ } FT_WORD; +typedef struct st_ftb_param { + byte prev; + int yesno; + int plusminus; + bool pmsign; + bool trunc; + byte *quot; +} FTB_PARAM; + int is_stopword(char *word, uint len); uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t); -TREE * ft_parse(TREE *, byte *, int); -FT_WORD * ft_linearize(MI_INFO *, uint, byte *, TREE *); +byte ft_get_word(byte **, byte *, FT_WORD *, FTB_PARAM *); +byte ft_simple_get_word(byte **, byte *, FT_WORD *); + +typedef struct _st_ft_seg_iterator { + uint num, len; + MI_KEYSEG *seg; + const byte *rec, *pos; +} FT_SEG_ITERATOR; + +void _mi_ft_segiterator_init(MI_INFO *, uint, const byte *, FT_SEG_ITERATOR *); +void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *); +uint _mi_ft_segiterator(FT_SEG_ITERATOR *); + +void ft_parse_init(TREE *, CHARSET_INFO *); +int ft_parse(TREE *, byte *, int); +FT_WORD * ft_linearize(TREE *); +FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, byte *, const byte *); +uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record); + +extern const struct _ft_vft _ft_vft_nlq; +FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, my_bool); +int ft_nlq_read_next(FT_INFO *, char *); +float ft_nlq_find_relevance(FT_INFO *, byte *, uint); +void ft_nlq_close_search(FT_INFO *); +float ft_nlq_get_relevance(FT_INFO *); +my_off_t ft_nlq_get_docid(FT_INFO *); +void ft_nlq_reinit_search(FT_INFO *); + +extern const struct _ft_vft _ft_vft_boolean; +FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, my_bool); +int ft_boolean_read_next(FT_INFO *, char *); +float ft_boolean_find_relevance(FT_INFO *, byte *, uint); +void ft_boolean_close_search(FT_INFO *); +float ft_boolean_get_relevance(FT_INFO *); +my_off_t ft_boolean_get_docid(FT_INFO *); +void ft_boolean_reinit_search(FT_INFO *); diff --git a/myisam/fulltext.h b/myisam/fulltext.h index 8fcac8172b1..f787c9bcfe8 100644 --- a/myisam/fulltext.h +++ b/myisam/fulltext.h @@ -24,7 +24,6 @@ /* shoudn't be def'ed when linking with mysql */ #undef EVAL_RUN -#define HA_FT_MAXLEN 254 #define HA_FT_WTYPE HA_KEYTYPE_FLOAT #define HA_FT_WLEN 4 #ifdef EVAL_RUN diff --git a/myisam/mi_cache.c b/myisam/mi_cache.c index f88725ff04b..8dee068c50e 100644 --- a/myisam/mi_cache.c +++ b/myisam/mi_cache.c @@ -1,27 +1,40 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Functions for read record cacheing with myisam */ -/* Used instead of my_b_read() to allow for no-cacheed seeks */ +/* + Functions for read record cacheing with myisam + Used for reading dynamic/compressed records from datafile. -#include "myisamdef.h" + Can fetch data directly from file (outside cache), + if reading a small chunk straight before the cached part (with possible + overlap). + + Can be explicitly asked not to use cache (by not setting READING_NEXT in + flag) - useful for occasional out-of-cache reads, when the next read is + expected to hit the cache again. - /* Copy block from cache if it`s in it. If re_read_if_possibly is */ - /* set read to cache (if after current file-position) else read to */ - /* buff */ + Allows "partial read" errors in the record header (when READING_HEADER flag + is set) - unread part is bzero'ed + + Note: out-of-cache reads are enabled for shared IO_CACHE's too, + as these reads will be cached by OS cache (and my_pread is always atomic) +*/ + + +#include "myisamdef.h" int _mi_read_cache(IO_CACHE *info, byte *buff, my_off_t pos, uint length, int flag) @@ -44,12 +57,13 @@ int _mi_read_cache(IO_CACHE *info, byte *buff, my_off_t pos, uint length, pos+=read_length; buff+=read_length; } - if ((offset= (my_off_t) (pos - info->pos_in_file)) < - (my_off_t) (info->rc_end - info->rc_request_pos)) + if (pos >= info->pos_in_file && + (offset= (my_off_t) (pos - info->pos_in_file)) < + (my_off_t) (info->read_end - info->request_pos)) { - in_buff_pos=info->rc_request_pos+(uint) offset; - in_buff_length= min(length,(uint) (info->rc_end-in_buff_pos)); - memcpy(buff,info->rc_request_pos+(uint) offset,(size_t) in_buff_length); + in_buff_pos=info->request_pos+(uint) offset; + in_buff_length= min(length,(uint) (info->read_end-in_buff_pos)); + memcpy(buff,info->request_pos+(uint) offset,(size_t) in_buff_length); if (!(length-=in_buff_length)) DBUG_RETURN(0); pos+=in_buff_length; @@ -60,38 +74,35 @@ int _mi_read_cache(IO_CACHE *info, byte *buff, my_off_t pos, uint length, if (flag & READING_NEXT) { if (pos != (info->pos_in_file + - (uint) (info->rc_end - info->rc_request_pos))) + (uint) (info->read_end - info->request_pos))) { info->pos_in_file=pos; /* Force start here */ - info->rc_pos=info->rc_end=info->rc_request_pos; /* Everything used */ + info->read_pos=info->read_end=info->request_pos; /* Everything used */ info->seek_not_done=1; } else - info->rc_pos=info->rc_end; /* All block used */ + info->read_pos=info->read_end; /* All block used */ if (!(*info->read_function)(info,buff,length)) DBUG_RETURN(0); - if (!(flag & READING_HEADER) || info->error == -1 || - (uint) info->error+in_buff_length < 3) - { - if (!my_errno || my_errno == -1) - my_errno=HA_ERR_WRONG_IN_RECORD; - DBUG_RETURN(1); - } - bzero(buff+info->error,MI_BLOCK_INFO_HEADER_LENGTH - in_buff_length - - (uint) info->error); - DBUG_RETURN(0); + read_length=info->error; + } + else + { + info->seek_not_done=1; + if ((read_length=my_pread(info->file,buff,length,pos,MYF(0))) == length) + DBUG_RETURN(0); } - info->seek_not_done=1; - if ((read_length=my_pread(info->file,buff,length,pos,MYF(0))) == length) - DBUG_RETURN(0); if (!(flag & READING_HEADER) || (int) read_length == -1 || read_length+in_buff_length < 3) { + DBUG_PRINT("error", + ("Error %d reading next-multi-part block (Got %d bytes)", + my_errno, (int) read_length)); if (!my_errno || my_errno == -1) my_errno=HA_ERR_WRONG_IN_RECORD; DBUG_RETURN(1); } bzero(buff+read_length,MI_BLOCK_INFO_HEADER_LENGTH - in_buff_length - - read_length); + read_length); DBUG_RETURN(0); } /* _mi_read_cache */ diff --git a/myisam/mi_changed.c b/myisam/mi_changed.c index bd6b14b0c6c..c2ab5568eba 100644 --- a/myisam/mi_changed.c +++ b/myisam/mi_changed.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -24,7 +24,7 @@ int mi_is_changed(MI_INFO *info) { int result; DBUG_ENTER("mi_is_changed"); - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(-1); VOID(_mi_writeinfo(info,0)); result=(int) info->data_changed; diff --git a/myisam/mi_check.c b/myisam/mi_check.c index 89fcfe74cea..ca5c8f9ecb4 100644 --- a/myisam/mi_check.c +++ b/myisam/mi_check.c @@ -14,14 +14,14 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Descript, check and repair of ISAM tables */ +/* Describe, check and repair of MyISAM tables */ -#include "fulltext.h" +#include "ftdefs.h" #include <m_ctype.h> #include <stdarg.h> -#include <getopt.h> +#include <my_getopt.h> #include <assert.h> -#ifdef HAVE_SYS_VADVICE_H +#ifdef HAVE_SYS_VADVISE_H #include <sys/vadvise.h> #endif #ifdef HAVE_SYS_MMAN_H @@ -45,25 +45,22 @@ static int writekeys(MI_CHECK *param, MI_INFO *info,byte *buff, my_off_t filepos); static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo, my_off_t pagepos, File new_file); -static int sort_key_read(SORT_INFO *sort_info,void *key); -static int sort_get_next_record(SORT_INFO *sort_info); -static int sort_key_cmp(SORT_INFO *sort_info, const void *a,const void *b); -static int sort_key_write(SORT_INFO *sort_info, const void *a); +static int sort_key_read(MI_SORT_PARAM *sort_param,void *key); +static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key); +static int sort_get_next_record(MI_SORT_PARAM *sort_param); +static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b); +static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a); static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo, uchar *key); -static int sort_insert_key(MI_CHECK *param, reg1 SORT_KEY_BLOCKS *key_block, +static int sort_insert_key(MI_SORT_PARAM *sort_param, + reg1 SORT_KEY_BLOCKS *key_block, uchar *key, my_off_t prev_block); -static int sort_delete_record(MI_CHECK *param); -static int flush_pending_blocks(MI_CHECK *param); +static int sort_delete_record(MI_SORT_PARAM *sort_param); +/*static int flush_pending_blocks(MI_CHECK *param);*/ static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks, uint buffer_length); -static void update_key_parts(MI_KEYDEF *keyinfo, - ulong *rec_per_key_part, - ulonglong *unique, - ulonglong records); static ha_checksum mi_byte_checksum(const byte *buf, uint length); -static void set_data_file_type(MI_CHECK *param, SORT_INFO *info, - MYISAM_SHARE *share); +static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share); #ifdef __WIN__ static double ulonglong2double(ulonglong value) @@ -79,7 +76,7 @@ static double ulonglong2double(ulonglong value) #else #define my_off_t2double(A) ((double) (A)) #endif /* SIZEOF_OFF_T > 4 */ -#endif +#endif /* __WIN__ */ void myisamchk_init(MI_CHECK *param) { @@ -95,7 +92,6 @@ void myisamchk_init(MI_CHECK *param) param->sort_key_blocks=BUFFERS_WHEN_SORTING; param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL); - param->sort_info.param=param; param->start_check_pos=0; } @@ -104,6 +100,7 @@ void myisamchk_init(MI_CHECK *param) int chk_status(MI_CHECK *param, register MI_INFO *info) { MYISAM_SHARE *share=info->s; + if (mi_is_crashed_on_repair(info)) mi_check_print_warning(param, "Table is marked as crashed and last repair failed"); @@ -219,7 +216,7 @@ int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag) } DBUG_RETURN(0); wrong: - param->retry_without_quick=1; /* Don't use quick repair */ + param->testflag|=T_RETRY_WITHOUT_QUICK; if (test_flag & T_VERBOSE) puts(""); mi_check_print_error(param,"record delete-link-chain corrupted"); DBUG_RETURN(1); @@ -231,7 +228,7 @@ wrong: static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr) { my_off_t next_link; - uint block_size=(nr+1)*MI_KEY_BLOCK_LENGTH; + uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH; ha_rows records; char llbuff[21],*buff; DBUG_ENTER("check_k_link"); @@ -319,7 +316,7 @@ int chk_size(MI_CHECK *param, register MI_INFO *info) error=1; mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s", llstr(size,buff), llstr(skr,buff2)); - param->retry_without_quick=1; /* Don't use quick repair */ + param->testflag|=T_RETRY_WITHOUT_QUICK; } else { @@ -371,8 +368,8 @@ int chk_key(MI_CHECK *param, register MI_INFO *info) { /* Remember old statistics for key */ memcpy((char*) rec_per_key_part, - (char*) share->state.rec_per_key_part+ - (uint) (rec_per_key_part - param->rec_per_key_part), + (char*) (share->state.rec_per_key_part + + (uint) (rec_per_key_part - param->rec_per_key_part)), keyinfo->keysegs*sizeof(*rec_per_key_part)); continue; } @@ -458,7 +455,7 @@ int chk_key(MI_CHECK *param, register MI_INFO *info) info->s->state.auto_increment=save_auto_value; /* Check that there isn't a row with auto_increment = 0 in the table */ - mi_extra(info,HA_EXTRA_KEYREAD); + mi_extra(info,HA_EXTRA_KEYREAD,0); bzero(info->lastkey,keyinfo->seg->length); if (!mi_rkey(info, info->rec_buff, key, (const byte*) info->lastkey, keyinfo->seg->length, HA_READ_KEY_EXACT)) @@ -469,7 +466,7 @@ int chk_key(MI_CHECK *param, register MI_INFO *info) "Found row where the auto_increment column has the value 0"); param->warning_printed=save; } - mi_extra(info,HA_EXTRA_NO_KEYREAD); + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); } length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2; @@ -604,7 +601,8 @@ static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, if (*keys != 1L) /* not first_key */ { uint diff; - _mi_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,SEARCH_FIND, + _mi_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY, + SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, &diff); param->unique_count[diff-1]++; } @@ -843,7 +841,8 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) } if (info->s->base.blobs) { - if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + if (!(to= mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) { mi_check_print_error(param,"Not enough memory for blob at %s", llstr(start_recpos,llbuff)); @@ -901,7 +900,7 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) info->checksum=mi_checksum(info,record); if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE)) { - if (_mi_rec_check(info,record)) + if (_mi_rec_check(info,record, info->rec_buff)) { mi_check_print_error(param,"Found wrong packed record at %s", llstr(start_recpos,llbuff)); @@ -921,7 +920,7 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) goto err; start_recpos=pos; splits++; - VOID(_mi_pack_get_block_info(info,&block_info, -1, start_recpos, NullS)); + VOID(_mi_pack_get_block_info(info,&block_info, -1, start_recpos)); pos=block_info.filepos+block_info.rec_len; if (block_info.rec_len < (uint) info->s->min_pack_length || block_info.rec_len > (uint) info->s->max_pack_length) @@ -1068,7 +1067,7 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) if (used != 0 && ! param->error_printed) { printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n", - llstr(records,llbuff), (long)((used-link_used)/records), + llstr(records,llbuff), (long)((used-link_used)/records), (info->s->base.blobs ? 0.0 : (ulonglong2double((ulonglong) info->s->base.reclength*records)- my_off_t2double(used))/ @@ -1092,7 +1091,7 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff)); err2: my_free((gptr) record,MYF(0)); - param->retry_without_quick=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; DBUG_RETURN(1); } /* chk_data_link */ @@ -1110,38 +1109,45 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, File new_file; MYISAM_SHARE *share=info->s; char llbuff[22],llbuff2[22]; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO sort_info; + MI_SORT_PARAM sort_param; DBUG_ENTER("mi_repair"); - sort_info->buff=sort_info->record=0; + bzero((char *)&sort_info, sizeof(sort_info)); + bzero((char *)&sort_param, sizeof(sort_param)); start_records=info->state->records; new_header_length=(param->testflag & T_UNPACK) ? 0L : share->pack.header_length; got_error=1; new_file= -1; + sort_param.sort_info=&sort_info; + if (!(param->testflag & T_SILENT)) { printf("- recovering (with keycache) MyISAM-table '%s'\n",name); printf("Data records: %s\n", llstr(info->state->records,llbuff)); } + param->testflag|=T_REP; /* for easy checking */ if (!param->using_global_keycache) - VOID(init_key_cache(param->use_buffers,NEAD_MEM)); + VOID(init_key_cache(param->use_buffers)); if (init_io_cache(¶m->read_cache,info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))) + { + bzero(&info->rec_cache,sizeof(info->rec_cache)); goto err; + } if (!rep_quick) if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL))) goto err; info->opt_flag|=WRITE_CACHE_USED; - sort_info->start_recpos=0; - sort_info->buff=0; sort_info->buff_length=0; - if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, - MYF(0)))) + if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength, + MYF(0))) || + !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) { mi_check_print_error(param,"Not enough memory for extra record"); goto err; @@ -1149,9 +1155,11 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, if (!rep_quick) { - if ((new_file=my_raid_create(fn_format(param->temp_filename,name,"", - DATA_TMP_EXT, - 2+4), + /* Get real path for data file */ + fn_format(param->temp_filename,name,"", MI_NAME_DEXT,2+4+32); + if ((new_file=my_raid_create(fn_format(param->temp_filename, + param->temp_filename,"", + DATA_TMP_EXT, 2+4), 0,param->tmpfile_createflag, share->base.raid_type, share->base.raid_chunks, @@ -1173,16 +1181,19 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, mi_int2store(share->state.header.options,share->options); } } - sort_info->info=info; - sort_info->pos=sort_info->max_pos=share->pack.header_length; - sort_info->filepos=new_header_length; - param->read_cache.end_of_file=sort_info->filelength= + sort_info.info=info; + sort_info.param = param; + sort_param.read_cache=param->read_cache; + sort_param.pos=sort_param.max_pos=share->pack.header_length; + sort_param.filepos=new_header_length; + param->read_cache.end_of_file=sort_info.filelength= my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)); - sort_info->dupp=0; - sort_info->fix_datafile= (my_bool) (! rep_quick); - sort_info->max_records= ~(ha_rows) 0; + sort_info.dupp=0; + sort_param.fix_datafile= (my_bool) (! rep_quick); + sort_param.master=1; + sort_info.max_records= ~(ha_rows) 0; - set_data_file_type(param, sort_info, share); + set_data_file_type(&sort_info, share); del=info->state->del; info->state->records=info->state->del=share->state.split=0; info->state->empty=0; @@ -1196,10 +1207,11 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, for (i=0 ; i < share->state.header.max_block_size ; i++) share->state.key_del[i]= HA_OFFSET_ERROR; - /* I think mi_repair and mi_repair_by_sort should do the same - (according, e.g. to ha_myisam::repair), but as mi_repair doesn't - touch key_map it cannot be used to T_CREATE_MISSING_KEYS. - That is the next line for... (serg) + /* + I think mi_repair and mi_repair_by_sort should do the same + (according, e.g. to ha_myisam::repair), but as mi_repair doesn't + touch key_map it cannot be used to T_CREATE_MISSING_KEYS. + That is what the next line is for... (serg) */ share->state.key_map= ((((ulonglong) 1L << share->base.keys)-1) & @@ -1208,36 +1220,37 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, info->state->key_file_length=share->base.keystart; lock_memory(param); /* Everything is alloced */ - while (!(error=sort_get_next_record(sort_info))) + while (!(error=sort_get_next_record(&sort_param))) { - if (writekeys(param, info,(byte*) sort_info->record,sort_info->filepos)) + if (writekeys(param,info,(byte*)sort_param.record,sort_param.filepos)) { if (my_errno != HA_ERR_FOUND_DUPP_KEY) goto err; - DBUG_DUMP("record",(byte*) sort_info->record,share->base.pack_reclength); + DBUG_DUMP("record",(byte*) sort_param.record,share->base.pack_reclength); mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s", info->errkey+1, - llstr(sort_info->start_recpos,llbuff), + llstr(sort_param.start_recpos,llbuff), llstr(info->dupp_key_pos,llbuff2)); if (param->testflag & T_VERBOSE) { VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey, - sort_info->record,0L)); + sort_param.record,0L)); _mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey, USE_WHOLE_KEY); } - sort_info->dupp++; - if (rep_quick == 1) + sort_info.dupp++; + if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK) { - param->error_printed=param->retry_without_quick=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + param->error_printed=1; goto err; } continue; } - if (sort_write_record(sort_info)) + if (sort_write_record(&sort_param)) goto err; } - if (error > 0 || write_data_suffix(param,info) || + if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) || flush_io_cache(&info->rec_cache) || param->read_cache.error < 0) goto err; @@ -1245,7 +1258,7 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, { VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); } - if (my_chsize(share->kfile,info->state->key_file_length,MYF(0))) + if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0))) { mi_check_print_warning(param, "Can't change size of indexfile, error: %d", @@ -1253,12 +1266,13 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, goto err; } - if (rep_quick && del+sort_info->dupp != info->state->del) + if (rep_quick && del+sort_info.dupp != info->state->del) { mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); mi_check_print_error(param,"Run recovery again without -q"); got_error=1; - param->retry_repair=param->retry_without_quick=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; goto err; } if (param->testflag & T_SAFE_REPAIR) @@ -1276,14 +1290,12 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, { my_close(info->dfile,MYF(0)); info->dfile=new_file; - info->state->data_file_length=sort_info->filepos; - /* Only whole records */ - share->state.split=info->state->records+info->state->del; + info->state->data_file_length=sort_param.filepos; share->state.version=(ulong) time((time_t*) 0); /* Force reopen */ } else { - info->state->data_file_length=sort_info->max_pos; + info->state->data_file_length=sort_param.max_pos; } if (param->testflag & T_CALC_CHECKSUM) share->state.checksum=param->glob_crc; @@ -1292,10 +1304,10 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, { if (start_records != info->state->records) printf("Data records: %s\n", llstr(info->state->records,llbuff)); - if (sort_info->dupp) + if (sort_info.dupp) mi_check_print_warning(param, "%s records have been removed", - llstr(sort_info->dupp,llbuff)); + llstr(sort_info.dupp,llbuff)); } got_error=0; @@ -1311,11 +1323,11 @@ err: { my_close(new_file,MYF(0)); info->dfile=new_file= -1; - if (change_to_newfile(share->filename,MI_NAME_DEXT, + if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, DATA_TMP_EXT, share->base.raid_chunks, (param->testflag & T_BACKUP_DATA ? MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || - mi_open_datafile(info,share)) + mi_open_datafile(info,share,-1)) got_error=1; } } @@ -1323,7 +1335,7 @@ err: { if (! param->error_printed) mi_check_print_error(param,"%d for record at pos %s",my_errno, - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param.start_recpos,llbuff)); if (new_file >= 0) { VOID(my_close(new_file,MYF(0))); @@ -1332,8 +1344,10 @@ err: } mi_mark_crashed_on_repair(info); } - my_free(sort_info->record,MYF(MY_ALLOW_ZERO_PTR)); - my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); + my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff), + MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR)); VOID(end_io_cache(¶m->read_cache)); info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); VOID(end_io_cache(&info->rec_cache)); @@ -1342,7 +1356,7 @@ err: { share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; share->pack.header_length=0; - share->data_file_type=sort_info->new_data_file_type; + share->data_file_type=sort_info.new_data_file_type; } share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES | STATE_NOT_ANALYZED); @@ -1501,8 +1515,10 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name) if (!(param->testflag & T_SILENT)) printf("- Sorting index for MyISAM-table '%s'\n",name); - if ((new_file=my_create(fn_format(param->temp_filename,name,"", - INDEX_TMP_EXT,2+4), + /* Get real path for index file */ + fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32); + if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename, + "", INDEX_TMP_EXT,2+4), 0,param->tmpfile_createflag,MYF(0))) <= 0) { mi_check_print_error(param,"Can't create new tempfile: '%s'", @@ -1522,7 +1538,7 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name) if (share->state.key_root[key] != HA_OFFSET_ERROR) { - index_pos[key]=param->new_file_pos; /* Write first block here */ + index_pos[key]=param->new_file_pos; /* Write first block here */ if (sort_one_index(param,info,keyinfo,share->state.key_root[key], new_file)) goto err; @@ -1538,12 +1554,12 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name) old_state=share->state; /* save state if not stored */ r_locks=share->r_locks; w_locks=share->w_locks; /* Put same locks as old file */ - share->r_locks=share->w_locks=0; + share->r_locks= share->w_locks= share->tot_locks= 0; (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); VOID(my_close(share->kfile,MYF(MY_WME))); share->kfile = -1; VOID(my_close(new_file,MYF(MY_WME))); - if (change_to_newfile(share->filename,MI_NAME_IEXT,INDEX_TMP_EXT,0, + if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0, MYF(0)) || mi_open_keyfile(share)) goto err2; @@ -1551,6 +1567,7 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name) _mi_readinfo(info,F_WRLCK,0); /* Will lock the table */ info->lock_type=F_WRLCK; share->r_locks=r_locks; share->w_locks=w_locks; + share->tot_locks= r_locks+w_locks; share->state=old_state; /* Restore old state */ info->state->key_file_length=param->new_file_pos; @@ -1643,9 +1660,14 @@ err: } /* sort_one_index */ - /* Change to use new file */ - /* Copy stats from old file to new file, deletes orginal and */ - /* changes new file name to old file name */ + /* + Let temporary file replace old file. + This assumes that the new file was created in the same + directory as given by realpath(filename). + This will ensure that any symlinks that are used will still work. + Copy stats from old file to new file, deletes orignal and + changes new file name to old file name + */ int change_to_newfile(const char * filename, const char * old_ext, const char * new_ext, @@ -1660,8 +1682,10 @@ int change_to_newfile(const char * filename, const char * old_ext, raid_chunks, MYF(MY_WME | MY_LINK_WARNING | MyFlags)); #endif - return my_redel(fn_format(old_filename,filename,"",old_ext,2+4), - fn_format(new_filename,filename,"",new_ext,2+4), + /* Get real path to filename */ + (void) fn_format(old_filename,filename,"",old_ext,2+4+32); + return my_redel(old_filename, + fn_format(new_filename,old_filename,"",new_ext,2+4), MYF(MY_WME | MY_LINK_WARNING | MyFlags)); } /* change_to_newfile */ @@ -1721,8 +1745,21 @@ err: DBUG_RETURN(1); } - /* Fix table or given index using sorting */ - /* saves new table in temp_filename */ + +/* + Repair table or given index using sorting + + SYNOPSIS + mi_repair_by_sort() + param Repair parameters + info MyISAM handler to repair + name Name of table (for warnings) + rep_quick set to <> 0 if we should not change data file + + RESULT + 0 ok + <>0 Error +*/ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, const char * name, int rep_quick) @@ -1735,9 +1772,10 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, File new_file; MI_SORT_PARAM sort_param; MYISAM_SHARE *share=info->s; + MI_KEYSEG *keyseg; ulong *rec_per_key_part; char llbuff[22]; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO sort_info; ulonglong key_map=share->state.key_map; DBUG_ENTER("mi_repair_by_sort"); @@ -1751,25 +1789,11 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, printf("- recovering (with sort) MyISAM-table '%s'\n",name); printf("Data records: %s\n", llstr(start_records,llbuff)); } + param->testflag|=T_REP; /* for easy checking */ - /* Hmm, repair_by_sort uses find_all_keys, and find_all_keys strictly - implies "one row - one key per keynr", while for ft_key one row/keynr - can produce as many keys as the number of unique words in the text - that's why I disabled repair_by_sort for ft-keys. (serg) - */ - for (i=0 ; i < share->base.keys ; i++) - { - if ((((ulonglong) 1 << i) & key_map) && - (share->keyinfo[i].flag & HA_FULLTEXT)) - { - mi_check_print_error(param, - "Can`t use repair_by_sort with FULLTEXT key"); - DBUG_RETURN(1); - } - } - - bzero((char*) sort_info,sizeof(*sort_info)); - if (!(sort_info->key_block= + bzero((char*)&sort_info,sizeof(sort_info)); + bzero((char *)&sort_param, sizeof(sort_param)); + if (!(sort_info.key_block= alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length)) @@ -1782,19 +1806,23 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw))) goto err; - sort_info->key_block_end=sort_info->key_block+param->sort_key_blocks; + sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks; info->opt_flag|=WRITE_CACHE_USED; info->rec_cache.file=info->dfile; /* for sort_delete_record */ - if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, - MYF(0)))) + if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength, + MYF(0))) || + !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) { mi_check_print_error(param,"Not enough memory for extra record"); goto err; } if (!rep_quick) { - if ((new_file=my_raid_create(fn_format(param->temp_filename,name,"", + /* Get real path for data file */ + fn_format(param->temp_filename,name,"", MI_NAME_DEXT,2+4+32); + if ((new_file=my_raid_create(fn_format(param->temp_filename, + param->temp_filename, "", DATA_TMP_EXT, 2+4), 0,param->tmpfile_createflag, @@ -1841,32 +1869,34 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, key_map= ~key_map; /* Create the missing keys */ } - sort_info->info=info; - sort_info->param = param; + sort_info.info=info; + sort_info.param = param; - set_data_file_type(param, sort_info, share); - sort_info->filepos=new_header_length; - sort_info->dupp=0; - sort_info->buff=0; - param->read_cache.end_of_file=sort_info->filelength= + set_data_file_type(&sort_info, share); + sort_param.filepos=new_header_length; + sort_info.dupp=0; + sort_info.buff=0; + param->read_cache.end_of_file=sort_info.filelength= my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0)); + sort_param.wordlist=NULL; + if (share->data_file_type == DYNAMIC_RECORD) length=max(share->base.min_pack_length+1,share->base.min_block_length); else if (share->data_file_type == COMPRESSED_RECORD) length=share->base.min_block_length; else length=share->base.pack_reclength; - sort_param.max_records=sort_info->max_records= - ((param->testflag & T_TRUST_HEADER) ? info->state->records : - (ha_rows) (sort_info->filelength/length+1)); + sort_info.max_records= + ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records : + (ha_rows) (sort_info.filelength/length+1)); sort_param.key_cmp=sort_key_cmp; sort_param.key_write=sort_key_write; - sort_param.key_read=sort_key_read; sort_param.lock_in_memory=lock_memory; sort_param.tmpdir=param->tmpdir; - sort_param.myf_rw=param->myf_rw; - sort_param.sort_info=sort_info; + sort_param.sort_info=&sort_info; + sort_param.fix_datafile= (my_bool) (! rep_quick); + sort_param.master =1; del=info->state->del; param->glob_crc=0; @@ -1874,40 +1904,51 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, param->calc_checksum=1; rec_per_key_part= param->rec_per_key_part; - for (sort_info->key=0 ; sort_info->key < share->base.keys ; - rec_per_key_part+=sort_info->keyinfo->keysegs, sort_info->key++) + for (sort_param.key=0 ; sort_param.key < share->base.keys ; + rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++) { - sort_info->keyinfo=share->keyinfo+sort_info->key; - if (!(((ulonglong) 1 << sort_info->key) & key_map)) + sort_param.read_cache=param->read_cache; + sort_param.keyinfo=share->keyinfo+sort_param.key; + if (!(((ulonglong) 1 << sort_param.key) & key_map)) { /* Remember old statistics for key */ memcpy((char*) rec_per_key_part, - (char*) share->state.rec_per_key_part+ - (uint) (rec_per_key_part - param->rec_per_key_part), - sort_info->keyinfo->keysegs*sizeof(*rec_per_key_part)); + (char*) (share->state.rec_per_key_part + + (uint) (rec_per_key_part - param->rec_per_key_part)), + sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part)); continue; } if ((!(param->testflag & T_SILENT))) - printf ("- Fixing index %d\n",sort_info->key+1); - sort_info->max_pos=sort_info->pos=share->pack.header_length; - sort_info->keyseg=sort_info->keyinfo->seg; - sort_info->fix_datafile= (my_bool) (sort_info->key == 0 && ! rep_quick); - bzero((char*) sort_info->unique,sizeof(sort_info->unique)); + printf ("- Fixing index %d\n",sort_param.key+1); + sort_param.max_pos=sort_param.pos=share->pack.header_length; + keyseg=sort_param.keyinfo->seg; + bzero((char*) sort_param.unique,sizeof(sort_param.unique)); sort_param.key_length=share->rec_reflength; - for (i=0 ; sort_info->keyseg[i].type != HA_KEYTYPE_END; i++) + for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++) { - sort_param.key_length+=sort_info->keyseg[i].length; - if (sort_info->keyseg[i].flag & HA_SPACE_PACK) - sort_param.key_length+=get_pack_length(sort_info->keyseg[i].length); - if (sort_info->keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH)) - sort_param.key_length+=2 + test(sort_info->keyseg[i].length >= 127); - if (sort_info->keyseg[i].flag & HA_NULL_PART) + sort_param.key_length+=keyseg[i].length; + if (keyseg[i].flag & HA_SPACE_PACK) + sort_param.key_length+=get_pack_length(keyseg[i].length); + if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH)) + sort_param.key_length+=2 + test(keyseg[i].length >= 127); + if (keyseg[i].flag & HA_NULL_PART) sort_param.key_length++; } info->state->records=info->state->del=share->state.split=0; info->state->empty=0; + if (sort_param.keyinfo->flag & HA_FULLTEXT) + { + sort_info.max_records= + (ha_rows) (sort_info.filelength/ft_max_word_len_for_sort+1); + + sort_param.key_read=sort_ft_key_read; + sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXLEN; + } + else + sort_param.key_read=sort_key_read; + if (_create_index_by_sort(&sort_param, (my_bool) (!(param->testflag & T_VERBOSE)), (uint) param->sort_buffer_length)) @@ -1918,18 +1959,17 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, param->calc_checksum=0; /* No need to calc glob_crc */ /* Set for next loop */ - sort_param.max_records=sort_info->max_records= - (ha_rows) info->state->records; + sort_info.max_records= (ha_rows) info->state->records; if (param->testflag & T_STATISTICS) - update_key_parts(sort_info->keyinfo, rec_per_key_part, sort_info->unique, + update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique, (ulonglong) info->state->records); - share->state.key_map|=(ulonglong) 1 << sort_info->key; + share->state.key_map|=(ulonglong) 1 << sort_param.key; - if (sort_info->fix_datafile) + if (sort_param.fix_datafile) { - param->read_cache.end_of_file=sort_info->filepos; - if (write_data_suffix(param,info) || end_io_cache(&info->rec_cache)) + param->read_cache.end_of_file=sort_param.filepos; + if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache)) goto err; if (param->testflag & T_SAFE_REPAIR) { @@ -1940,25 +1980,25 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, goto err; } } - share->state.state.data_file_length = info->state->data_file_length - = sort_info->filepos; + share->state.state.data_file_length = info->state->data_file_length= + sort_param.filepos; /* Only whole records */ - share->state.split=info->state->records+info->state->del; share->state.version=(ulong) time((time_t*) 0); my_close(info->dfile,MYF(0)); info->dfile=new_file; - share->data_file_type=sort_info->new_data_file_type; + share->data_file_type=sort_info.new_data_file_type; share->pack.header_length=(ulong) new_header_length; + sort_param.fix_datafile=0; } else - info->state->data_file_length=sort_info->max_pos; + info->state->data_file_length=sort_param.max_pos; - if (flush_pending_blocks(param)) - goto err; + /*if (flush_pending_blocks(param)) + goto err;*/ param->read_cache.file=info->dfile; /* re-init read cache */ - reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length,1, - 1); + reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length, + 1,1); } if (param->testflag & T_WRITE_LOOP) @@ -1966,16 +2006,17 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); } - if (rep_quick && del+sort_info->dupp != info->state->del) + if (rep_quick && del+sort_info.dupp != info->state->del) { mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); mi_check_print_error(param,"Run recovery again without -q"); got_error=1; - param->retry_repair=param->retry_without_quick=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; goto err; } - if (rep_quick != 1) + if (rep_quick & T_FORCE_UNIQUENESS) { my_off_t skr=info->state->data_file_length+ (share->options & HA_OPTION_COMPRESS_RECORD ? @@ -1985,8 +2026,8 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, skr < share->base.reloc*share->base.min_pack_length) skr=share->base.reloc*share->base.min_pack_length; #endif - if (skr != sort_info->filelength && !info->s->base.raid_type) - if (my_chsize(info->dfile,skr,MYF(0))) + if (skr != sort_info.filelength && !info->s->base.raid_type) + if (my_chsize(info->dfile,skr,0,MYF(0))) mi_check_print_warning(param, "Can't change size of datafile, error: %d", my_errno); @@ -1994,7 +2035,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, if (param->testflag & T_CALC_CHECKSUM) share->state.checksum=param->glob_crc; - if (my_chsize(share->kfile,info->state->key_file_length,MYF(0))) + if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0))) mi_check_print_warning(param, "Can't change size of indexfile, error: %d", my_errno); @@ -2003,10 +2044,10 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, { if (start_records != info->state->records) printf("Data records: %s\n", llstr(info->state->records,llbuff)); - if (sort_info->dupp) + if (sort_info.dupp) mi_check_print_warning(param, "%s records have been removed", - llstr(sort_info->dupp,llbuff)); + llstr(sort_info.dupp,llbuff)); } got_error=0; @@ -2023,11 +2064,11 @@ err: { my_close(new_file,MYF(0)); info->dfile=new_file= -1; - if (change_to_newfile(share->filename,MI_NAME_DEXT, + if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, DATA_TMP_EXT, share->base.raid_chunks, (param->testflag & T_BACKUP_DATA ? MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || - mi_open_datafile(info,share)) + mi_open_datafile(info,share,-1)) got_error=1; } } @@ -2049,9 +2090,11 @@ err: share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS; share->state.changed|=STATE_NOT_SORTED_PAGES; - my_free((gptr) sort_info->key_block,MYF(MY_ALLOW_ZERO_PTR)); - my_free(sort_info->record,MYF(MY_ALLOW_ZERO_PTR)); - my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); + my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff), + MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR)); VOID(end_io_cache(¶m->read_cache)); info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); if (!got_error && (param->testflag & T_UNPACK)) @@ -2062,74 +2105,519 @@ err: DBUG_RETURN(got_error); } +/* + Threaded repair of table using sorting + + SYNOPSIS + mi_repair_by_sort_r() + param Repair parameters + info MyISAM handler to repair + name Name of table (for warnings) + rep_quick set to <> 0 if we should not change data file + + DESCRIPTION + Same as mi_repair_by_sort but do it multithreaded + Each key is handled by a separate thread. + TODO: make a number of threads a parameter + + RESULT + 0 ok + <>0 Error +*/ + +int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, + const char * name, int rep_quick) +{ + int got_error; + uint i,key, total_key_length, istep; + ulong rec_length; + ha_rows start_records; + my_off_t new_header_length,del; + File new_file; + MI_SORT_PARAM *sort_param=0; + MYISAM_SHARE *share=info->s; + ulong *rec_per_key_part; + MI_KEYSEG *keyseg; + char llbuff[22]; + IO_CACHE_SHARE io_share; + SORT_INFO sort_info; + ulonglong key_map=share->state.key_map; + pthread_attr_t thr_attr; + DBUG_ENTER("mi_repair_parallel"); + + start_records=info->state->records; + got_error=1; + new_file= -1; + new_header_length=(param->testflag & T_UNPACK) ? 0 : + share->pack.header_length; + if (!(param->testflag & T_SILENT)) + { + printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(start_records,llbuff)); + } + param->testflag|=T_REP; /* for easy checking */ + + bzero((char*)&sort_info,sizeof(sort_info)); + if (!(sort_info.key_block= + alloc_key_blocks(param, + (uint) param->sort_key_blocks, + share->base.max_key_block_length)) + || init_io_cache(¶m->read_cache,info->dfile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) || + (! rep_quick && + init_io_cache(&info->rec_cache,info->dfile, + (uint) param->write_buffer_length, + WRITE_CACHE,new_header_length,1, + MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw))) + goto err; + sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks; + info->opt_flag|=WRITE_CACHE_USED; + info->rec_cache.file=info->dfile; /* for sort_delete_record */ + + if (!rep_quick) + { + /* Get real path for data file */ + fn_format(param->temp_filename,name,"", MI_NAME_DEXT,2+4+32); + if ((new_file=my_raid_create(fn_format(param->temp_filename, + param->temp_filename, "", + DATA_TMP_EXT, + 2+4), + 0,param->tmpfile_createflag, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (filecopy(param, new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + share->state.dellink= HA_OFFSET_ERROR; + info->rec_cache.file=new_file; + } + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + if (!(param->testflag & T_CREATE_MISSING_KEYS)) + { + /* + Flush key cache for this file if we are calling this outside + myisamchk + */ + flush_key_blocks(share->kfile, FLUSH_IGNORE_CHANGED); + /* Clear the pointers to the given rows */ + for (i=0 ; i < share->base.keys ; i++) + share->state.key_root[i]= HA_OFFSET_ERROR; + for (i=0 ; i < share->state.header.max_block_size ; i++) + share->state.key_del[i]= HA_OFFSET_ERROR; + info->state->key_file_length=share->base.keystart; + } + else + { + if (flush_key_blocks(share->kfile, FLUSH_FORCE_WRITE)) + goto err; + key_map= ~key_map; /* Create the missing keys */ + } + + sort_info.info=info; + sort_info.param = param; + + set_data_file_type(&sort_info, share); + sort_info.dupp=0; + sort_info.buff=0; + param->read_cache.end_of_file=sort_info.filelength= + my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0)); + + if (share->data_file_type == DYNAMIC_RECORD) + rec_length=max(share->base.min_pack_length+1,share->base.min_block_length); + else if (share->data_file_type == COMPRESSED_RECORD) + rec_length=share->base.min_block_length; + else + rec_length=share->base.pack_reclength; + sort_info.max_records= + ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records : + (ha_rows) (sort_info.filelength/rec_length+1)); + + del=info->state->del; + param->glob_crc=0; + if (param->testflag & T_CALC_CHECKSUM) + param->calc_checksum=1; + + if (!(sort_param=(MI_SORT_PARAM *) + my_malloc((uint) share->base.keys * + (sizeof(MI_SORT_PARAM) + share->base.pack_reclength), + MYF(MY_ZEROFILL)))) + { + mi_check_print_error(param,"Not enough memory!"); + goto err; + } + total_key_length=0; + rec_per_key_part= param->rec_per_key_part; + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + + for (i=key=0, istep=1 ; key < share->base.keys ; + rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++) + { + sort_param[i].key=key; + sort_param[i].keyinfo=share->keyinfo+key; + if (!(((ulonglong) 1 << key) & key_map)) + { + /* Remember old statistics for key */ + memcpy((char*) rec_per_key_part, + (char*) (share->state.rec_per_key_part+ + (uint) (rec_per_key_part - param->rec_per_key_part)), + sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part)); + istep=0; + continue; + } + istep=1; + if ((!(param->testflag & T_SILENT))) + printf ("- Fixing index %d\n",key+1); + sort_param[i].key_read= ((sort_param[i].keyinfo->flag & HA_FULLTEXT) ? + sort_ft_key_read : sort_key_read); + sort_param[i].key_cmp=sort_key_cmp; + sort_param[i].key_write=sort_key_write; + sort_param[i].lock_in_memory=lock_memory; + sort_param[i].tmpdir=param->tmpdir; + sort_param[i].sort_info=&sort_info; + sort_param[i].master=0; + sort_param[i].fix_datafile=0; + + sort_param[i].filepos=new_header_length; + sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length; + + sort_param[i].record= (((char *)(sort_param+share->base.keys))+ + (share->base.pack_reclength * i)); + + sort_param[i].key_length=share->rec_reflength; + for (keyseg=sort_param[i].keyinfo->seg; keyseg->type != HA_KEYTYPE_END; + keyseg++) + { + sort_param[i].key_length+=keyseg->length; + if (keyseg->flag & HA_SPACE_PACK) + sort_param[i].key_length+=get_pack_length(keyseg->length); + if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH)) + sort_param[i].key_length+=2 + test(keyseg->length >= 127); + if (keyseg->flag & HA_NULL_PART) + sort_param[i].key_length++; + } + total_key_length+=sort_param[i].key_length; + + if (sort_param[i].keyinfo->flag & HA_FULLTEXT) + sort_param[i].key_length+=ft_max_word_len_for_sort-ft_max_word_len; + } + sort_info.total_keys=i; + sort_param[0].master= 1; + sort_param[0].fix_datafile= (my_bool)(! rep_quick); + + sort_info.got_error=0; + pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST); + pthread_cond_init(&sort_info.cond, 0); + pthread_mutex_lock(&sort_info.mutex); + + init_io_cache_share(¶m->read_cache, &io_share, i); + (void) pthread_attr_init(&thr_attr); + (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED); + + for (i=0 ; i < sort_info.total_keys ; i++) + { + sort_param[i].read_cache=param->read_cache; + /* + two approaches: the same amount of memory for each thread + or the memory for the same number of keys for each thread... + In the second one all the threads will fill their sort_buffers + (and call write_keys) at the same time, putting more stress on i/o. + */ + sort_param[i].sortbuff_size= +#ifndef USING_SECOND_APPROACH + param->sort_buffer_length/sort_info.total_keys; +#else + param->sort_buffer_length*sort_param[i].key_length/total_key_length; +#endif + if (pthread_create(&sort_param[i].thr, &thr_attr, + thr_find_all_keys, + (void *) (sort_param+i))) + { + mi_check_print_error(param,"Cannot start a repair thread"); + remove_io_thread(¶m->read_cache); + sort_info.got_error=1; + } + else + sort_info.threads_running++; + } + (void) pthread_attr_destroy(&thr_attr); + + /* waiting for all threads to finish */ + while (sort_info.threads_running) + pthread_cond_wait(&sort_info.cond, &sort_info.mutex); + pthread_mutex_unlock(&sort_info.mutex); + + if ((got_error= thr_write_keys(sort_param))) + { + param->retry_repair=1; + goto err; + } + got_error=1; /* Assume the following may go wrong */ + + if (sort_param[0].fix_datafile) + { + if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache)) + goto err; + if (param->testflag & T_SAFE_REPAIR) + { + /* Don't repair if we loosed more than one row */ + if (info->state->records+1 < start_records) + { + info->state->records=start_records; + goto err; + } + } + share->state.state.data_file_length= info->state->data_file_length= + sort_param->filepos; + /* Only whole records */ + share->state.version=(ulong) time((time_t*) 0); + my_close(info->dfile,MYF(0)); + info->dfile=new_file; + share->data_file_type=sort_info.new_data_file_type; + share->pack.header_length=(ulong) new_header_length; + } + else + info->state->data_file_length=sort_param->max_pos; + + if (rep_quick && del+sort_info.dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without -q"); + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + goto err; + } + + if (rep_quick & T_FORCE_UNIQUENESS) + { + my_off_t skr=info->state->data_file_length+ + (share->options & HA_OPTION_COMPRESS_RECORD ? + MEMMAP_EXTRA_MARGIN : 0); +#ifdef USE_RELOC + if (share->data_file_type == STATIC_RECORD && + skr < share->base.reloc*share->base.min_pack_length) + skr=share->base.reloc*share->base.min_pack_length; +#endif + if (skr != sort_info.filelength && !info->s->base.raid_type) + if (my_chsize(info->dfile,skr,0,MYF(0))) + mi_check_print_warning(param, + "Can't change size of datafile, error: %d", + my_errno); + } + if (param->testflag & T_CALC_CHECKSUM) + share->state.checksum=param->glob_crc; + + if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0))) + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", my_errno); + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info.dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info.dupp,llbuff)); + } + got_error=0; + + if (&share->state.state != info->state) + memcpy(&share->state.state, info->state, sizeof(*info->state)); + +err: + got_error|= flush_blocks(param,share->kfile); + VOID(end_io_cache(&info->rec_cache)); + if (!got_error) + { + /* Replace the actual file with the temporary file */ + if (new_file >= 0) + { + my_close(new_file,MYF(0)); + info->dfile=new_file= -1; + if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, + DATA_TMP_EXT, share->base.raid_chunks, + (param->testflag & T_BACKUP_DATA ? + MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + mi_open_datafile(info,share,-1)) + got_error=1; + } + } + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d when fixing table",my_errno); + if (new_file >= 0) + { + VOID(my_close(new_file,MYF(0))); + VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks, + MYF(MY_WME))); + if (info->dfile == new_file) + info->dfile= -1; + } + mi_mark_crashed_on_repair(info); + } + else if (key_map == share->state.key_map) + share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS; + share->state.changed|=STATE_NOT_SORTED_PAGES; + + pthread_cond_destroy (&sort_info.cond); + pthread_mutex_destroy(&sort_info.mutex); + + my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr) sort_param,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR)); + VOID(end_io_cache(¶m->read_cache)); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + if (!got_error && (param->testflag & T_UNPACK)) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + } + DBUG_RETURN(got_error); +} /* Read next record and return next key */ -static int sort_key_read(SORT_INFO *sort_info, void *key) +static int sort_key_read(MI_SORT_PARAM *sort_param, void *key) { int error; - MI_INFO *info; + SORT_INFO *sort_info=sort_param->sort_info; + MI_INFO *info=sort_info->info; DBUG_ENTER("sort_key_read"); - info=sort_info->info; - - if ((error=sort_get_next_record(sort_info))) + if ((error=sort_get_next_record(sort_param))) DBUG_RETURN(error); if (info->state->records == sort_info->max_records) { mi_check_print_error(sort_info->param, - "Found too many records; Can`t continue"); + "Key %d - Found too many records; Can't continue", + sort_param->key+1); DBUG_RETURN(1); } - (void) _mi_make_key(info,sort_info->key,(uchar*)key,sort_info->record, - sort_info->filepos); - DBUG_RETURN(sort_write_record(sort_info)); + sort_param->real_key_length= + (info->s->rec_reflength+ + _mi_make_key(info, sort_param->key, (uchar*) key, + sort_param->record, sort_param->filepos)); +#ifdef HAVE_purify + bzero(key+sort_param->real_key_length, + (sort_param->key_length-sort_param->real_key_length)); +#endif + DBUG_RETURN(sort_write_record(sort_param)); } /* sort_key_read */ +static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key) +{ + int error; + SORT_INFO *sort_info=sort_param->sort_info; + MI_INFO *info=sort_info->info; + FT_WORD *wptr=0; + DBUG_ENTER("sort_ft_key_read"); + + if (!sort_param->wordlist) + { + for (;;) + { + my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR)); + if ((error=sort_get_next_record(sort_param))) + DBUG_RETURN(error); + if (!(wptr=_mi_ft_parserecord(info,sort_param->key, + key,sort_param->record))) + DBUG_RETURN(1); + if (wptr->pos) + break; + error=sort_write_record(sort_param); + } + sort_param->wordptr=sort_param->wordlist=wptr; + } + else + { + error=0; + wptr=(FT_WORD*)(sort_param->wordptr); + } + + sort_param->real_key_length=(info->s->rec_reflength+ + _ft_make_key(info, sort_param->key, + key, wptr++, sort_param->filepos)); +#ifdef HAVE_purify + if (sort_param->key_length > sort_param->real_key_length) + bzero(key+sort_param->real_key_length, + (sort_param->key_length-sort_param->real_key_length)); +#endif + if (!wptr->pos) + { + my_free((char*) sort_param->wordlist, MYF(0)); + sort_param->wordlist=0; + error=sort_write_record(sort_param); + } + else + sort_param->wordptr=(void*)wptr; + + DBUG_RETURN(error); +} /* sort_ft_key_read */ + + /* Read next record from file using parameters in sort_info */ /* Return -1 if end of file, 0 if ok and > 0 if error */ -static int sort_get_next_record(SORT_INFO *sort_info) +static int sort_get_next_record(MI_SORT_PARAM *sort_param) { int searching; uint found_record,b_type,left_length; my_off_t pos; byte *to; MI_BLOCK_INFO block_info; - MI_INFO *info; - MYISAM_SHARE *share; + SORT_INFO *sort_info=sort_param->sort_info; MI_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; char llbuff[22],llbuff2[22]; DBUG_ENTER("sort_get_next_record"); - info=sort_info->info; - share=info->s; switch (share->data_file_type) { case STATIC_RECORD: for (;;) { - if (my_b_read(¶m->read_cache,sort_info->record, + if (my_b_read(&sort_param->read_cache,sort_param->record, share->base.pack_reclength)) { - if (param->read_cache.error) + if (sort_param->read_cache.error) param->out_flag |= O_DATA_LOST; - param->retry_repair=param->retry_without_quick=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; DBUG_RETURN(-1); } - sort_info->start_recpos=sort_info->pos; - if (!sort_info->fix_datafile) - sort_info->filepos=sort_info->pos; - sort_info->max_pos=(sort_info->pos+=share->base.pack_reclength); - share->state.split++; - if (*sort_info->record) + sort_param->start_recpos=sort_param->pos; + if (!sort_param->fix_datafile) + { + sort_param->filepos=sort_param->pos; + if (sort_param->master) + share->state.split++; + } + sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength); + if (*sort_param->record) { if (param->calc_checksum) param->glob_crc+= (info->checksum= - mi_static_checksum(info,sort_info->record)); + mi_static_checksum(info,sort_param->record)); DBUG_RETURN(0); } - if (!sort_info->fix_datafile) + if (!sort_param->fix_datafile && sort_param->master) { info->state->del++; info->state->empty+=share->base.pack_reclength; @@ -2137,8 +2625,8 @@ static int sort_get_next_record(SORT_INFO *sort_info) } case DYNAMIC_RECORD: LINT_INIT(to); - pos=sort_info->pos; - searching=(sort_info->fix_datafile && (param->testflag & T_EXTEND)); + pos=sort_param->pos; + searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND)); for (;;) { found_record=block_info.second_read= 0; @@ -2146,13 +2634,13 @@ static int sort_get_next_record(SORT_INFO *sort_info) if (searching) { pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE); - param->retry_without_quick=1; - sort_info->start_recpos=pos; + param->testflag|=T_RETRY_WITHOUT_QUICK; + sort_param->start_recpos=pos; } do { - if (pos > sort_info->max_pos) - sort_info->max_pos=pos; + if (pos > sort_param->max_pos) + sort_param->max_pos=pos; if (pos & (MI_DYN_ALIGN_SIZE-1)) { if ((param->testflag & T_VERBOSE) || searching == 0) @@ -2164,8 +2652,9 @@ static int sort_get_next_record(SORT_INFO *sort_info) if (found_record && pos == param->search_after_block) mi_check_print_info(param,"Block: %s used by record at %s", llstr(param->search_after_block,llbuff), - llstr(sort_info->start_recpos,llbuff2)); - if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header,pos, + llstr(sort_param->start_recpos,llbuff2)); + if (_mi_read_cache(&sort_param->read_cache, + (byte*) block_info.header,pos, MI_BLOCK_INFO_HEADER_LENGTH, (! found_record ? READING_NEXT : 0) | READING_HEADER)) @@ -2174,20 +2663,21 @@ static int sort_get_next_record(SORT_INFO *sort_info) { mi_check_print_info(param, "Can't read whole record at %s (errno: %d)", - llstr(sort_info->start_recpos,llbuff),errno); + llstr(sort_param->start_recpos,llbuff),errno); goto try_next; } DBUG_RETURN(-1); } - if (searching && ! sort_info->fix_datafile) + if (searching && ! sort_param->fix_datafile) { param->error_printed=1; - param->retry_repair=param->retry_without_quick=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; DBUG_RETURN(1); /* Something wrong with data */ } - if (((b_type=_mi_get_block_info(&block_info,-1,pos)) & - (BLOCK_ERROR | BLOCK_FATAL_ERROR)) || - ((b_type & BLOCK_FIRST) && + b_type=_mi_get_block_info(&block_info,-1,pos); + if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) || + ((b_type & BLOCK_FIRST) && (block_info.rec_len < (uint) share->base.min_pack_length || block_info.rec_len > (uint) share->base.max_pack_length))) { @@ -2209,7 +2699,7 @@ static int sort_get_next_record(SORT_INFO *sort_info) block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE) break; pos+=(ulong) i; - sort_info->start_recpos=pos; + sort_param->start_recpos=pos; continue; } if (b_type & BLOCK_DELETED) @@ -2245,7 +2735,7 @@ static int sort_get_next_record(SORT_INFO *sort_info) goto try_next; searching=1; pos+= MI_DYN_ALIGN_SIZE; - sort_info->start_recpos=pos; + sort_param->start_recpos=pos; block_info.second_read=0; continue; } @@ -2266,14 +2756,15 @@ static int sort_get_next_record(SORT_INFO *sort_info) goto try_next; searching=1; pos+= MI_DYN_ALIGN_SIZE; - sort_info->start_recpos=pos; + sort_param->start_recpos=pos; block_info.second_read=0; continue; } } if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) { - if (!sort_info->fix_datafile && (b_type & BLOCK_DELETED)) + if (!sort_param->fix_datafile && sort_param->master && + (b_type & BLOCK_DELETED)) { info->state->empty+=block_info.block_len; info->state->del++; @@ -2284,7 +2775,7 @@ static int sort_get_next_record(SORT_INFO *sort_info) if (searching) { pos+=MI_DYN_ALIGN_SIZE; - sort_info->start_recpos=pos; + sort_param->start_recpos=pos; } else pos=block_info.filepos+block_info.block_len; @@ -2292,43 +2783,45 @@ static int sort_get_next_record(SORT_INFO *sort_info) continue; } - share->state.split++; + if (!sort_param->fix_datafile && sort_param->master) + share->state.split++; if (! found_record++) { - sort_info->find_length=left_length=block_info.rec_len; - sort_info->start_recpos=pos; - if (!sort_info->fix_datafile) - sort_info->filepos=sort_info->start_recpos; - if (sort_info->fix_datafile && (param->testflag & T_EXTEND)) - sort_info->pos=block_info.filepos+1; + sort_param->find_length=left_length=block_info.rec_len; + sort_param->start_recpos=pos; + if (!sort_param->fix_datafile) + sort_param->filepos=sort_param->start_recpos; + if (sort_param->fix_datafile && (param->testflag & T_EXTEND)) + sort_param->pos=block_info.filepos+1; else - sort_info->pos=block_info.filepos+block_info.block_len; + sort_param->pos=block_info.filepos+block_info.block_len; if (share->base.blobs) { - if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + if (!(to=mi_alloc_rec_buff(info,block_info.rec_len, + &(sort_param->rec_buff)))) { mi_check_print_error(param,"Not enough memory for blob at %s", - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); DBUG_RETURN(1); } } else - to= info->rec_buff; + to= sort_param->rec_buff; } if (left_length < block_info.data_len || ! block_info.data_len) { mi_check_print_info(param,"Found block with too small length at %s; Skipped", - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); goto try_next; } if (block_info.filepos + block_info.data_len > - param->read_cache.end_of_file) + sort_param->read_cache.end_of_file) { mi_check_print_info(param,"Found block that points outside data file at %s", - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); goto try_next; } - if (_mi_read_cache(¶m->read_cache,to,block_info.filepos, + if (_mi_read_cache(&sort_param->read_cache,to,block_info.filepos, block_info.data_len, (found_record == 1 ? READING_NEXT : 0))) { @@ -2343,31 +2836,31 @@ static int sort_get_next_record(SORT_INFO *sort_info) if (pos == HA_OFFSET_ERROR && left_length) { mi_check_print_info(param,"Wrong block with wrong total length starting at %s", - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); goto try_next; } - if (pos + MI_BLOCK_INFO_HEADER_LENGTH > param->read_cache.end_of_file) + if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file) { mi_check_print_info(param,"Found link that points at %s (outside data file) at %s", llstr(pos,llbuff2), - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); goto try_next; } } while (left_length); - if (_mi_rec_unpack(info,sort_info->record,info->rec_buff, - sort_info->find_length) != MY_FILE_ERROR) + if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff, + sort_param->find_length) != MY_FILE_ERROR) { - if (param->read_cache.error < 0) + if (sort_param->read_cache.error < 0) DBUG_RETURN(1); if (info->s->calc_checksum) - info->checksum=mi_checksum(info,sort_info->record); + info->checksum=mi_checksum(info,sort_param->record); if ((param->testflag & (T_EXTEND | T_REP)) || searching) { - if (_mi_rec_check(info, sort_info->record)) + if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff)) { mi_check_print_info(param,"Found wrong packed record at %s", - llstr(sort_info->start_recpos,llbuff)); + llstr(sort_param->start_recpos,llbuff)); goto try_next; } } @@ -2376,31 +2869,33 @@ static int sort_get_next_record(SORT_INFO *sort_info) DBUG_RETURN(0); } if (!searching) - mi_check_print_info(param,"Found wrong stored record at %s", - llstr(sort_info->start_recpos,llbuff)); + mi_check_print_info(param,"Key %d - Found wrong stored record at %s", + sort_param->key+1, + llstr(sort_param->start_recpos,llbuff)); try_next: - pos=(sort_info->start_recpos+=MI_DYN_ALIGN_SIZE); + pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE); searching=1; } case COMPRESSED_RECORD: - for (searching=0 ;; searching=1, sort_info->pos++) + for (searching=0 ;; searching=1, sort_param->pos++) { - if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header, - sort_info->pos, + if (_mi_read_cache(&sort_param->read_cache,(byte*) block_info.header, + sort_param->pos, share->pack.ref_length,READING_NEXT)) DBUG_RETURN(-1); - if (searching && ! sort_info->fix_datafile) + if (searching && ! sort_param->fix_datafile) { param->error_printed=1; - param->retry_repair=param->retry_without_quick=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; DBUG_RETURN(1); /* Something wrong with data */ } - sort_info->start_recpos=sort_info->pos; - if (_mi_pack_get_block_info(info,&block_info,-1,sort_info->pos, NullS)) + sort_param->start_recpos=sort_param->pos; + if (_mi_pack_get_block_info(info,&block_info,-1,sort_param->pos)) DBUG_RETURN(-1); if (!block_info.rec_len && - sort_info->pos + MEMMAP_EXTRA_MARGIN == - param->read_cache.end_of_file) + sort_param->pos + MEMMAP_EXTRA_MARGIN == + sort_param->read_cache.end_of_file) DBUG_RETURN(-1); if (block_info.rec_len < (uint) share->min_pack_length || block_info.rec_len > (uint) share->max_pack_length) @@ -2408,32 +2903,35 @@ static int sort_get_next_record(SORT_INFO *sort_info) if (! searching) mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n", block_info.rec_len, - llstr(sort_info->pos,llbuff)); + llstr(sort_param->pos,llbuff)); continue; } - if (_mi_read_cache(¶m->read_cache,(byte*) info->rec_buff, + if (_mi_read_cache(&sort_param->read_cache,(byte*) sort_param->rec_buff, block_info.filepos, block_info.rec_len, READING_NEXT)) { if (! searching) mi_check_print_info(param,"Couldn't read whole record from %s", - llstr(sort_info->pos,llbuff)); + llstr(sort_param->pos,llbuff)); continue; } - if (_mi_pack_rec_unpack(info,sort_info->record,info->rec_buff, + if (_mi_pack_rec_unpack(info,sort_param->record,sort_param->rec_buff, block_info.rec_len)) { if (! searching) mi_check_print_info(param,"Found wrong record at %s", - llstr(sort_info->pos,llbuff)); + llstr(sort_param->pos,llbuff)); continue; } - info->checksum=mi_checksum(info,sort_info->record); - if (!sort_info->fix_datafile) - sort_info->filepos=sort_info->pos; - sort_info->max_pos=(sort_info->pos=block_info.filepos+ + info->checksum=mi_checksum(info,sort_param->record); + if (!sort_param->fix_datafile) + { + sort_param->filepos=sort_param->pos; + if (sort_param->master) + share->state.split++; + } + sort_param->max_pos=(sort_param->pos=block_info.filepos+ block_info.rec_len); - share->state.split++; info->packed_length=block_info.rec_len; if (param->calc_checksum) param->glob_crc+= info->checksum; @@ -2446,41 +2944,41 @@ static int sort_get_next_record(SORT_INFO *sort_info) /* Write record to new file */ -int sort_write_record(SORT_INFO *sort_info) +int sort_write_record(MI_SORT_PARAM *sort_param) { int flag; uint length; ulong block_length,reclength; byte *from; byte block_buff[8]; - MI_INFO *info; - MYISAM_SHARE *share; + SORT_INFO *sort_info=sort_param->sort_info; MI_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; DBUG_ENTER("sort_write_record"); - info=sort_info->info; - share=info->s; - if (sort_info->fix_datafile) + if (sort_param->fix_datafile) { switch (sort_info->new_data_file_type) { case STATIC_RECORD: - if (my_b_write(&info->rec_cache,sort_info->record, + if (my_b_write(&info->rec_cache,sort_param->record, share->base.pack_reclength)) { mi_check_print_error(param,"%d when writing to datafile",my_errno); DBUG_RETURN(1); } - sort_info->filepos+=share->base.pack_reclength; - /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_info->record); */ + sort_param->filepos+=share->base.pack_reclength; + info->s->state.split++; + /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */ break; case DYNAMIC_RECORD: if (! info->blobs) - from=info->rec_buff; + from=sort_param->rec_buff; else { /* must be sure that local buffer is big enough */ reclength=info->s->base.pack_reclength+ - _my_calc_total_blob_length(info,sort_info->record)+ + _my_calc_total_blob_length(info,sort_param->record)+ ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ MI_DYN_DELETE_BLOCK_HEADER; if (sort_info->buff_length < reclength) @@ -2493,22 +2991,30 @@ int sort_write_record(SORT_INFO *sort_info) } from=sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER); } - info->checksum=mi_checksum(info,sort_info->record); - reclength=_mi_rec_pack(info,from,sort_info->record); - /* sort_info->param->glob_crc+=info->checksum; */ - block_length=reclength+ 3 + test(reclength >= (65520-3)); - if (block_length < share->base.min_block_length) - block_length=share->base.min_block_length; + info->checksum=mi_checksum(info,sort_param->record); + reclength=_mi_rec_pack(info,from,sort_param->record); flag=0; - info->update|=HA_STATE_WRITE_AT_END; - block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE); - if (_mi_write_part_record(info,0L,block_length,HA_OFFSET_ERROR, - &from,&reclength,&flag)) + /* sort_info->param->glob_crc+=info->checksum; */ + + do { - mi_check_print_error(param,"%d when writing to datafile",my_errno); - DBUG_RETURN(1); - } - sort_info->filepos+=block_length; + block_length=reclength+ 3 + test(reclength >= (65520-3)); + if (block_length < share->base.min_block_length) + block_length=share->base.min_block_length; + info->update|=HA_STATE_WRITE_AT_END; + block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE); + if (block_length > MI_MAX_BLOCK_LENGTH) + block_length=MI_MAX_BLOCK_LENGTH; + if (_mi_write_part_record(info,0L,block_length, + sort_param->filepos+block_length, + &from,&reclength,&flag)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_param->filepos+=block_length; + info->s->state.split++; + } while (reclength); /* sort_info->param->glob_crc+=info->checksum; */ break; case COMPRESSED_RECORD: @@ -2517,22 +3023,26 @@ int sort_write_record(SORT_INFO *sort_info) if (info->s->base.blobs) length+=save_pack_length(block_buff+length,info->blob_length); if (my_b_write(&info->rec_cache,block_buff,length) || - my_b_write(&info->rec_cache,(byte*) info->rec_buff,reclength)) + my_b_write(&info->rec_cache,(byte*) sort_param->rec_buff,reclength)) { mi_check_print_error(param,"%d when writing to datafile",my_errno); DBUG_RETURN(1); } /* sort_info->param->glob_crc+=info->checksum; */ - sort_info->filepos+=reclength+length; + sort_param->filepos+=reclength+length; + info->s->state.split++; break; } } - info->state->records++; - if ((param->testflag & T_WRITE_LOOP) && - (info->state->records % WRITE_COUNT) == 0) + if (sort_param->master) { - char llbuff[22]; - printf("%s\r", llstr(info->state->records,llbuff)); VOID(fflush(stdout)); + info->state->records++; + if ((param->testflag & T_WRITE_LOOP) && + (info->state->records % WRITE_COUNT) == 0) + { + char llbuff[22]; + printf("%s\r", llstr(info->state->records,llbuff)); VOID(fflush(stdout)); + } } DBUG_RETURN(0); } /* sort_write_record */ @@ -2540,49 +3050,49 @@ int sort_write_record(SORT_INFO *sort_info) /* Compare two keys from _create_index_by_sort */ -static int sort_key_cmp(SORT_INFO *sort_info, const void *a, const void *b) +static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, + const void *b) { uint not_used; - return (_mi_key_cmp(sort_info->keyseg,*((uchar**) a),*((uchar**) b), + return (_mi_key_cmp(sort_param->keyinfo->seg,*((uchar**) a),*((uchar**) b), USE_WHOLE_KEY, SEARCH_SAME,¬_used)); } /* sort_key_cmp */ -static int sort_key_write(SORT_INFO *sort_info, const void *a) +static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a) { uint diff_pos; char llbuff[22],llbuff2[22]; + SORT_INFO *sort_info=sort_param->sort_info; MI_CHECK *param= sort_info->param; int cmp; if (sort_info->key_block->inited) { - cmp=_mi_key_cmp(sort_info->keyseg,sort_info->key_block->lastkey,(uchar*) a, - USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE ,&diff_pos); - sort_info->unique[diff_pos-1]++; + cmp=_mi_key_cmp(sort_param->keyinfo->seg, sort_info->key_block->lastkey, + (uchar*) a, USE_WHOLE_KEY, SEARCH_FIND | SEARCH_UPDATE, + &diff_pos); + sort_param->unique[diff_pos-1]++; } else { cmp= -1; } - if ((sort_info->keyinfo->flag & HA_NOSAME) && cmp == 0) + if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0) { sort_info->dupp++; sort_info->info->lastpos=get_record_for_key(sort_info->info, - sort_info->keyinfo, + sort_param->keyinfo, (uchar*) a); mi_check_print_warning(param, - "Duplicate key for record at %10s against record at %10s", - llstr(sort_info->info->lastpos,llbuff), - llstr(get_record_for_key(sort_info->info, - sort_info->keyinfo, - sort_info->key_block-> - lastkey), - llbuff2)); - param->retry_without_quick=1; + "Duplicate key for record at %10s against record at %10s", + llstr(sort_info->info->lastpos,llbuff), + llstr(get_record_for_key(sort_info->info, sort_param->keyinfo, + sort_info->key_block->lastkey), llbuff2)); + param->testflag|=T_RETRY_WITHOUT_QUICK; if (sort_info->param->testflag & T_VERBOSE) - _mi_print_key(stdout,sort_info->keyseg,(uchar*) a, USE_WHOLE_KEY); - return (sort_delete_record(param)); + _mi_print_key(stdout,sort_param->keyinfo->seg,(uchar*) a, USE_WHOLE_KEY); + return (sort_delete_record(sort_param)); } #ifndef DBUG_OFF if (cmp > 0) @@ -2592,8 +3102,8 @@ static int sort_key_write(SORT_INFO *sort_info, const void *a) return(1); } #endif - return (sort_insert_key(param,sort_info->key_block,(uchar*) a, - HA_OFFSET_ERROR)); + return (sort_insert_key(sort_param,sort_info->key_block, + (uchar*) a, HA_OFFSET_ERROR)); } /* sort_key_write */ @@ -2608,7 +3118,7 @@ static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo, /* Insert a key in sort-key-blocks */ -static int sort_insert_key(MI_CHECK *param, +static int sort_insert_key(MI_SORT_PARAM *sort_param, register SORT_KEY_BLOCKS *key_block, uchar *key, my_off_t prev_block) { @@ -2617,14 +3127,16 @@ static int sort_insert_key(MI_CHECK *param, uchar *anc_buff,*lastkey; MI_KEY_PARAM s_temp; MI_INFO *info; - SORT_INFO *sort_info= ¶m->sort_info; + MI_KEYDEF *keyinfo=sort_param->keyinfo; + SORT_INFO *sort_info= sort_param->sort_info; + MI_CHECK *param=sort_info->param; DBUG_ENTER("sort_insert_key"); anc_buff=key_block->buff; info=sort_info->info; lastkey=key_block->lastkey; nod_flag= (key_block == sort_info->key_block ? 0 : - sort_info->info->s->base.key_reflength); + info->s->base.key_reflength); if (!key_block->inited) { @@ -2645,17 +3157,16 @@ static int sort_insert_key(MI_CHECK *param, if (nod_flag) _mi_kpointer(info,key_block->end_pos,prev_block); - t_length=(*sort_info->keyinfo->pack_key)(sort_info->keyinfo,nod_flag, - (uchar*) 0,lastkey,lastkey,key, - &s_temp); - (*sort_info->keyinfo->store_key)(sort_info->keyinfo, - key_block->end_pos+nod_flag,&s_temp); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (uchar*) 0,lastkey,lastkey,key, + &s_temp); + (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp); a_length+=t_length; mi_putint(anc_buff,a_length,nod_flag); key_block->end_pos+=t_length; - if (a_length <= sort_info->keyinfo->block_length) + if (a_length <= keyinfo->block_length) { - VOID(_mi_move_key(sort_info->keyinfo,key_block->lastkey,key)); + VOID(_mi_move_key(keyinfo,key_block->lastkey,key)); key_block->last_length=a_length-t_length; DBUG_RETURN(0); } @@ -2663,52 +3174,50 @@ static int sort_insert_key(MI_CHECK *param, /* Fill block with end-zero and write filled block */ mi_putint(anc_buff,key_block->last_length,nod_flag); bzero((byte*) anc_buff+key_block->last_length, - sort_info->keyinfo->block_length- key_block->last_length); + keyinfo->block_length- key_block->last_length); key_file_length=info->state->key_file_length; - if ((filepos=_mi_new(info,sort_info->keyinfo)) == HA_OFFSET_ERROR) + if ((filepos=_mi_new(info,keyinfo)) == HA_OFFSET_ERROR) DBUG_RETURN(1); /* If we read the page from the key cache, we have to write it back to it */ if (key_file_length == info->state->key_file_length) { - if (_mi_write_keypage(info, sort_info->keyinfo, filepos, - anc_buff)) + if (_mi_write_keypage(info, keyinfo, filepos, anc_buff)) DBUG_RETURN(1); } else if (my_pwrite(info->s->kfile,(byte*) anc_buff, - (uint) sort_info->keyinfo->block_length,filepos, - param->myf_rw)) + (uint) keyinfo->block_length,filepos, param->myf_rw)) DBUG_RETURN(1); DBUG_DUMP("buff",(byte*) anc_buff,mi_getint(anc_buff)); /* Write separator-key to block in next level */ - if (sort_insert_key(param,key_block+1,key_block->lastkey,filepos)) + if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos)) DBUG_RETURN(1); /* clear old block and write new key in it */ key_block->inited=0; - DBUG_RETURN(sort_insert_key(param, key_block,key,prev_block)); + DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block)); } /* sort_insert_key */ /* Delete record when we found a duplicated key */ -static int sort_delete_record(MI_CHECK *param) +static int sort_delete_record(MI_SORT_PARAM *sort_param) { uint i; int old_file,error; uchar *key; - MI_INFO *info; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO *sort_info=sort_param->sort_info; + MI_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; DBUG_ENTER("sort_delete_record"); - if (param->opt_rep_quick == 1) + if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK) { mi_check_print_error(param, - "Quick-recover aborted; Run recovery without switch 'q' or with switch -qq"); + "Quick-recover aborted; Run recovery without switch -q or with switch -qq"); DBUG_RETURN(1); } - info=sort_info->info; if (info->s->options & HA_OPTION_COMPRESS_RECORD) { mi_check_print_error(param, @@ -2718,10 +3227,10 @@ static int sort_delete_record(MI_CHECK *param) old_file=info->dfile; info->dfile=info->rec_cache.file; - if (sort_info->key) + if (sort_info->current_key) { key=info->lastkey+info->s->base.max_key_length; - if ((error=(*info->s->read_rnd)(info,sort_info->record,info->lastpos,0)) && + if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) && error != HA_ERR_RECORD_DELETED) { mi_check_print_error(param,"Can't read record to be removed"); @@ -2729,9 +3238,9 @@ static int sort_delete_record(MI_CHECK *param) DBUG_RETURN(1); } - for (i=0 ; i < sort_info->key ; i++) + for (i=0 ; i < sort_info->current_key ; i++) { - uint key_length=_mi_make_key(info,i,key,sort_info->record,info->lastpos); + uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos); if (_mi_ck_delete(info,i,key,key_length)) { mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1); @@ -2740,8 +3249,7 @@ static int sort_delete_record(MI_CHECK *param) } } if (param->calc_checksum) - param->glob_crc-=(*info->s->calc_checksum)(info, - sort_info->record); + param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record); } error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info); info->dfile=old_file; /* restore actual value */ @@ -2749,20 +3257,20 @@ static int sort_delete_record(MI_CHECK *param) DBUG_RETURN(error); } /* sort_delete_record */ - /* Fix all pending blocks and flush everything to disk */ -static int flush_pending_blocks(MI_CHECK *param) +int flush_pending_blocks(MI_SORT_PARAM *sort_param) { uint nod_flag,length; my_off_t filepos,key_file_length; - MI_INFO *info; SORT_KEY_BLOCKS *key_block; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO *sort_info= sort_param->sort_info; + MI_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + MI_KEYDEF *keyinfo=sort_param->keyinfo; DBUG_ENTER("flush_pending_blocks"); filepos= HA_OFFSET_ERROR; /* if empty file */ - info=sort_info->info; nod_flag=0; for (key_block=sort_info->key_block ; key_block->inited ; key_block++) { @@ -2771,30 +3279,26 @@ static int flush_pending_blocks(MI_CHECK *param) if (nod_flag) _mi_kpointer(info,key_block->end_pos,filepos); key_file_length=info->state->key_file_length; - bzero((byte*) key_block->buff+length, - sort_info->keyinfo->block_length-length); - if ((filepos=_mi_new(info,sort_info->keyinfo)) == HA_OFFSET_ERROR) + bzero((byte*) key_block->buff+length, keyinfo->block_length-length); + if ((filepos=_mi_new(info,keyinfo)) == HA_OFFSET_ERROR) DBUG_RETURN(1); /* If we read the page from the key cache, we have to write it back */ if (key_file_length == info->state->key_file_length) { - if (_mi_write_keypage(info, sort_info->keyinfo, filepos, - key_block->buff)) + if (_mi_write_keypage(info, keyinfo, filepos, key_block->buff)) DBUG_RETURN(1); } else if (my_pwrite(info->s->kfile,(byte*) key_block->buff, - (uint) sort_info->keyinfo->block_length,filepos, - param->myf_rw)) + (uint) keyinfo->block_length,filepos, param->myf_rw)) DBUG_RETURN(1); DBUG_DUMP("buff",(byte*) key_block->buff,length); nod_flag=1; } - info->s->state.key_root[sort_info->key]=filepos; /* Last is root for tree */ + info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */ DBUG_RETURN(0); } /* flush_pending_blocks */ - /* alloc space and pointers for key_blocks */ static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks, @@ -2846,7 +3350,6 @@ int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename) MI_STATUS_INFO status_info; uint unpack,key_parts; ha_rows max_records; - char name[FN_REFLEN]; ulonglong file_length,tmp_length; MI_CREATE_INFO create_info; @@ -2955,8 +3458,9 @@ int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename) create_info.language = (param->language ? param->language : share.state.header.language); - if (mi_create(fn_format(name,filename,"",MI_NAME_IEXT, - 4+ (param->opt_follow_links ? 16 : 0)), + /* We don't have to handle symlinks here because we are using + HA_DONT_TOUCH_DATA */ + if (mi_create(filename, share.base.keys - share.state.header.uniques, keyinfo, share.base.fields, recdef, share.state.header.uniques, uniquedef, @@ -2966,7 +3470,7 @@ int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename) mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno); goto end; } - *org_info=mi_open(name,O_RDWR, + *org_info=mi_open(filename,O_RDWR, (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED : (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED : HA_OPEN_ABORT_IF_LOCKED); @@ -3004,24 +3508,25 @@ end: /* write suffix to data file if neaded */ -int write_data_suffix(MI_CHECK *param, MI_INFO *info) +int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile) { - if (info->s->options & HA_OPTION_COMPRESS_RECORD && - param->sort_info.fix_datafile) + MI_INFO *info=sort_info->info; + + if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile) { char buff[MEMMAP_EXTRA_MARGIN]; bzero(buff,sizeof(buff)); if (my_b_write(&info->rec_cache,buff,sizeof(buff))) { - mi_check_print_error(param,"%d when writing to datafile",my_errno); + mi_check_print_error(sort_info->param, + "%d when writing to datafile",my_errno); return 1; } - param->read_cache.end_of_file+=sizeof(buff); + sort_info->param->read_cache.end_of_file+=sizeof(buff); } return 0; } - /* Update state and myisamchk_time of indexfile */ int update_state_info(MI_CHECK *param, MI_INFO *info,uint update) @@ -3057,9 +3562,11 @@ int update_state_info(MI_CHECK *param, MI_INFO *info,uint update) { /* Force update of status */ int error; uint r_locks=share->r_locks,w_locks=share->w_locks; - share->r_locks=share->w_locks=0; + share->r_locks= share->w_locks= share->tot_locks= 0; error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK); - share->r_locks=r_locks; share->w_locks=w_locks; + share->r_locks=r_locks; + share->w_locks=w_locks; + share->tot_locks=r_locks+w_locks; if (!error) return 0; } @@ -3095,15 +3602,15 @@ void update_auto_increment_key(MI_CHECK *param, MI_INFO *info, return; } if (!(param->testflag & T_SILENT) && - !(param->testflag & (T_REP | T_REP_BY_SORT))) + !(param->testflag & T_REP)) printf("Updating MyISAM file: %s\n", param->isam_file_name); /* We have to use keyread here as a normal read uses info->rec_buff */ - mi_extra(info,HA_EXTRA_KEYREAD); + mi_extra(info,HA_EXTRA_KEYREAD,0); if (mi_rlast(info,info->rec_buff, info->s->base.auto_key-1)) { if (my_errno != HA_ERR_END_OF_FILE) { - mi_extra(info,HA_EXTRA_NO_KEYREAD); + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); mi_check_print_error(param,"%d when reading last record",my_errno); return; } @@ -3118,17 +3625,15 @@ void update_auto_increment_key(MI_CHECK *param, MI_INFO *info, update_auto_increment(info,info->rec_buff); set_if_bigger(info->s->state.auto_increment,auto_increment); } - mi_extra(info,HA_EXTRA_NO_KEYREAD); + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); update_state_info(param, info, UPDATE_AUTO_INC); return; } /* calculate unique keys for each part key */ -static void update_key_parts(MI_KEYDEF *keyinfo, - ulong *rec_per_key_part, - ulonglong *unique, - ulonglong records) +void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part, + ulonglong *unique, ulonglong records) { ulonglong count=0,tmp; uint parts; @@ -3147,7 +3652,7 @@ static void update_key_parts(MI_KEYDEF *keyinfo, } -ha_checksum mi_byte_checksum(const byte *buf, uint length) +static ha_checksum mi_byte_checksum(const byte *buf, uint length) { ha_checksum crc; const byte *end=buf+length; @@ -3168,12 +3673,15 @@ ha_checksum mi_byte_checksum(const byte *buf, uint length) static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows) { + uint key_maxlength=key->maxlength; + if (key->flag & HA_FULLTEXT) + key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXLEN; return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) && - ((ulonglong) rows * key->maxlength > + ((ulonglong) rows * key_maxlength > (ulonglong) myisam_max_temp_length || - (ulonglong) rows * (key->maxlength - key->minlength) / 2 > + (ulonglong) rows * (key_maxlength - key->minlength) / 2 > myisam_max_extra_temp_length || - (rows == 0 && (key->maxlength / key->minlength) > 2))); + (rows == 0 && (key_maxlength / key->minlength) > 2))); } @@ -3186,8 +3694,8 @@ void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows) MI_KEYDEF *key=share->keyinfo; for (i=0 ; i < share->base.keys ; i++,key++) { - if (!(key->flag & HA_NOSAME) && ! mi_too_big_key_for_sort(key,rows) && - info->s->base.auto_key != i+1) + if (!(key->flag & (HA_NOSAME|HA_AUTO_KEY)) && + ! mi_too_big_key_for_sort(key,rows)) { share->state.key_map&= ~ ((ulonglong) 1 << i); info->update|= HA_STATE_CHANGED; @@ -3203,31 +3711,22 @@ void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows) even if the temporary file would be quite big! */ -my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows, - ulonglong key_map, - my_bool force) +my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows, + ulonglong key_map, my_bool force) { MYISAM_SHARE *share=info->s; MI_KEYDEF *key=share->keyinfo; uint i; /* - repair_by_sort only works if we have at least one key. If we don't + mi_repair_by_sort only works if we have at least one key. If we don't have any keys, we should use the normal repair. */ if (!key_map) return FALSE; /* Can't use sort */ for (i=0 ; i < share->base.keys ; i++,key++) { -/* It's to disable repair_by_sort for ft-keys. - Another solution would be to make ft-keys just too_big_key_for_sort, - but then they won't be disabled by dectivate_non_unique_index - and so they will be created at the first stage. As ft-key creation - is very time-consuming process, it's better to leave it to repair stage - but this repair shouldn't be repair_by_sort (serg) - */ - if ((!force && mi_too_big_key_for_sort(key,rows)) || - (key->flag & HA_FULLTEXT)) + if (!force && mi_too_big_key_for_sort(key,rows)) return FALSE; } return TRUE; @@ -3235,10 +3734,10 @@ my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows, static void -set_data_file_type(MI_CHECK *param, SORT_INFO *sort_info, MYISAM_SHARE *share) +set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share) { if ((sort_info->new_data_file_type=share->data_file_type) == - COMPRESSED_RECORD && param->testflag & T_UNPACK) + COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK) { MYISAM_SHARE tmp; diff --git a/myisam/mi_checksum.c b/myisam/mi_checksum.c index 9dabe55e239..a760b03a032 100644 --- a/myisam/mi_checksum.c +++ b/myisam/mi_checksum.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_close.c b/myisam/mi_close.c index f30119144bc..dbaaebb1143 100644 --- a/myisam/mi_close.c +++ b/myisam/mi_close.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -29,8 +29,7 @@ int mi_close(register MI_INFO *info) MYISAM_SHARE *share=info->s; DBUG_ENTER("mi_close"); DBUG_PRINT("enter",("base: %lx reopen: %u locks: %u", - info,(uint) share->reopen, - (uint) (share->w_locks+share->r_locks))); + info,(uint) share->reopen, (uint) share->tot_locks)); pthread_mutex_lock(&THR_LOCK_myisam); if (info->lock_type == F_EXTRA_LCK) @@ -47,7 +46,10 @@ int mi_close(register MI_INFO *info) pthread_mutex_lock(&share->intern_lock); if (share->options & HA_OPTION_READ_ONLY_DATA) + { share->r_locks--; + share->tot_locks--; + } if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) { if (end_io_cache(&info->rec_cache)) @@ -58,6 +60,7 @@ int mi_close(register MI_INFO *info) myisam_open_list=list_delete(myisam_open_list,&info->open_list); pthread_mutex_unlock(&share->intern_lock); + my_free(mi_get_rec_buff_ptr(info, info->rec_buff), MYF(MY_ALLOW_ZERO_PTR)); if (flag) { if (share->kfile >= 0 && @@ -99,7 +102,6 @@ int mi_close(register MI_INFO *info) error = my_errno; myisam_log_command(MI_LOG_CLOSE,info,NULL,0,error); - my_free((gptr) info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); my_free((gptr) info,MYF(0)); if (error) diff --git a/myisam/mi_create.c b/myisam/mi_create.c index d0ba37a77c6..9082c2b0d95 100644 --- a/myisam/mi_create.c +++ b/myisam/mi_create.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -38,12 +38,13 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, register uint i,j; File dfile,file; int errpos,save_errno; + myf create_flag; uint fields,length,max_key_length,packed,pointer, key_length,info_length,key_segs,options,min_key_length_skipp, base_pos,varchar_count,long_varchar_count,varchar_length, max_key_block_length,unique_key_parts,offset; ulong reclength, real_reclength,min_pack_length; - char buff[FN_REFLEN]; + char filename[FN_REFLEN],linkname[FN_REFLEN], *linkname_ptr; ulong pack_reclength; ulonglong tot_length,max_rows; enum en_fieldtype type; @@ -118,8 +119,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, pack_reclength+=(1 << ((rec->length-mi_portable_sizeof_char_ptr)*8)); /* Max blob length */ } } - else if (type == FIELD_SKIPP_PRESPACE || - type == FIELD_SKIPP_ENDSPACE) + else if (type == FIELD_SKIP_PRESPACE || + type == FIELD_SKIP_ENDSPACE) { if (pack_reclength != INT_MAX32) pack_reclength+= rec->length > 255 ? 2 : 1; @@ -137,7 +138,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, pack_reclength+=2; } } - else if (type != FIELD_SKIPP_ZERO) + else if (type != FIELD_SKIP_ZERO) { min_pack_length+=rec->length; packed--; /* Not a pack record type */ @@ -151,7 +152,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, while (rec != recinfo) { rec--; - if (rec->type == (int) FIELD_SKIPP_ZERO && rec->length == 1) + if (rec->type == (int) FIELD_SKIP_ZERO && rec->length == 1) { rec->type=(int) FIELD_NORMAL; packed--; @@ -163,6 +164,9 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, if (packed || (flags & HA_PACK_RECORD)) options|=HA_OPTION_PACK_RECORD; /* Must use packed records */ + /* We can't use checksum with static length rows */ + if (!(options & HA_OPTION_PACK_RECORD)) + options&= ~HA_OPTION_CHECKSUM; if (options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) min_pack_length+=varchar_count; /* Min length to pack */ else @@ -223,8 +227,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, share.state.key_del=key_del; if (uniques) { - max_key_block_length= MI_KEY_BLOCK_LENGTH; - max_key_length= MI_UNIQUE_HASH_LENGTH + pointer; + max_key_block_length= myisam_block_size; + max_key_length= MI_UNIQUE_HASH_LENGTH + pointer; } for (i=0, keydef=keydefs ; i < keys ; i++ , keydef++) @@ -300,7 +304,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, if (keydef->flag & HA_BINARY_PACK_KEY) options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ - if (keydef->flag & HA_AUTO_KEY) + if (keydef->flag & HA_AUTO_KEY && ci->with_auto_increment) share.base.auto_key=i+1; for (j=0, keyseg=keydef->seg ; j < keydef->keysegs ; j++, keyseg++) { @@ -366,7 +370,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, share.state.rec_per_key_part[key_segs-1]=1L; length+=key_length; keydef->block_length= MI_BLOCK_SIZE(length,pointer,MI_MAX_KEYPTR_SIZE); - if (keydef->block_length/MI_KEY_BLOCK_LENGTH > MI_MAX_KEY_BLOCK_SIZE) + if (keydef->block_length > MI_MAX_KEY_BLOCK_LENGTH || + length >= MI_MAX_KEY_BUFF) { my_errno=HA_WRONG_CREATE_OPTION; goto err; @@ -382,7 +387,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, (length*2)))* (ulong) keydef->block_length; } - for (i=max_key_block_length/MI_KEY_BLOCK_LENGTH ; i-- ; ) + for (i=max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH ; i-- ; ) key_del[i]=HA_OFFSET_ERROR; unique_key_parts=0; @@ -400,7 +405,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, key_segs+=uniques; /* Each unique has 1 key seg */ base_pos=(MI_STATE_INFO_SIZE + keys * MI_STATE_KEY_SIZE + - max_key_block_length/MI_KEY_BLOCK_LENGTH*MI_STATE_KEYBLOCK_SIZE+ + max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH* + MI_STATE_KEYBLOCK_SIZE+ key_segs*MI_STATE_KEYSEG_SIZE); info_length=base_pos+(uint) (MI_BASE_INFO_SIZE+ keys * MI_KEYDEF_SIZE+ @@ -419,7 +425,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, mi_int2store(share.state.header.base_pos,base_pos); share.state.header.language= (ci->language ? ci->language : MY_CHARSET_CURRENT); - share.state.header.max_block_size=max_key_block_length/MI_KEY_BLOCK_LENGTH; + share.state.header.max_block_size=max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH; share.state.dellink = HA_OFFSET_ERROR; share.state.process= (ulong) getpid(); @@ -432,7 +438,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, share.base.rec_reflength=pointer; share.base.key_reflength= mi_get_pointer_length((tot_length + max_key_block_length * keys * - MI_INDEX_BLOCK_MARGIN) / MI_KEY_BLOCK_LENGTH, + MI_INDEX_BLOCK_MARGIN) / MI_MIN_KEY_BLOCK_LENGTH, 3); share.base.keys= share.state.header.keys = keys; share.state.header.uniques= uniques; @@ -471,17 +477,40 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, if (! (flags & HA_DONT_TOUCH_DATA)) share.state.create_time= (long) time((time_t*) 0); - if ((file = my_create(fn_format(buff,name,"",MI_NAME_IEXT,4),0, - O_RDWR | O_TRUNC,MYF(MY_WME))) < 0) + if (ci->index_file_name) + { + fn_format(filename, ci->index_file_name,"",MI_NAME_IEXT,4); + fn_format(linkname,name, "",MI_NAME_IEXT,4); + linkname_ptr=linkname; + /* + Don't create the table if the link or file exists to ensure that one + doesn't accidently destroy another table. + */ + create_flag=0; + } + else + { + fn_format(filename,name,"",MI_NAME_IEXT,(4+ (flags & HA_DONT_TOUCH_DATA) ? + 32 : 0)); + linkname_ptr=0; + /* Replace the current file */ + create_flag=MY_DELETE_OLD; + } + + if ((file= my_create_with_symlink(linkname_ptr, + filename, + 0, O_RDWR | O_TRUNC, + MYF(MY_WME | create_flag))) < 0) goto err; errpos=1; - VOID(fn_format(buff,name,"",MI_NAME_DEXT,2+4)); + if (!(flags & HA_DONT_TOUCH_DATA)) { #ifdef USE_RAID if (share.base.raid_type) { - if ((dfile=my_raid_create(buff,0,O_RDWR | O_TRUNC, + (void) fn_format(filename,name,"",MI_NAME_DEXT,2+4); + if ((dfile=my_raid_create(filename,0,O_RDWR | O_TRUNC, share.base.raid_type, share.base.raid_chunks, share.base.raid_chunksize, @@ -490,9 +519,26 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, } else #endif - if ((dfile = my_create(buff,0,O_RDWR | O_TRUNC,MYF(MY_WME))) < 0) - goto err; - + { + if (ci->data_file_name) + { + fn_format(filename, ci->data_file_name,"",MI_NAME_DEXT,4); + fn_format(linkname, name, "",MI_NAME_DEXT,4); + linkname_ptr=linkname; + create_flag=0; + } + else + { + fn_format(filename,name,"",MI_NAME_DEXT,4); + linkname_ptr=0; + create_flag=MY_DELETE_OLD; + } + if ((dfile= + my_create_with_symlink(linkname_ptr, filename, + 0,O_RDWR | O_TRUNC, + MYF(MY_WME | create_flag))) < 0) + goto err; + } errpos=3; } @@ -511,14 +557,14 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, /* Write key and keyseg definitions */ for (i=0 ; i < share.base.keys - uniques; i++) { - uint ft_segs=(keydefs[i].flag & HA_FULLTEXT) ? FT_SEGS : 0; /* SerG */ + uint ft_segs=(keydefs[i].flag & HA_FULLTEXT) ? FT_SEGS : 0; if (mi_keydef_write(file, &keydefs[i])) goto err; for (j=0 ; j < keydefs[i].keysegs-ft_segs ; j++) if (mi_keyseg_write(file, &keydefs[i].seg[j])) goto err; - for (j=0 ; j < ft_segs ; j++) /* SerG */ + for (j=0 ; j < ft_segs ; j++) { MI_KEYSEG seg=ft_keysegs[j]; seg.language= keydefs[i].seg[0].language; @@ -534,11 +580,11 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, { tmp_keydef.keysegs=1; tmp_keydef.flag= HA_UNIQUE_CHECK; - tmp_keydef.block_length= MI_KEY_BLOCK_LENGTH; + tmp_keydef.block_length= myisam_block_size; tmp_keydef.keylength= MI_UNIQUE_HASH_LENGTH + pointer; tmp_keydef.minlength=tmp_keydef.maxlength=tmp_keydef.keylength; - tmp_keyseg.type= MI_UNIQUE_HASH_TYPE; - tmp_keyseg.length= MI_UNIQUE_HASH_LENGTH; + tmp_keyseg.type= MI_UNIQUE_HASH_TYPE; + tmp_keyseg.length= MI_UNIQUE_HASH_LENGTH; tmp_keyseg.start= offset; offset+= MI_UNIQUE_HASH_LENGTH; if (mi_keydef_write(file,&tmp_keydef) || @@ -571,13 +617,13 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, #endif /* Enlarge files */ - if (my_chsize(file,(ulong) share.base.keystart,MYF(0))) + if (my_chsize(file,(ulong) share.base.keystart,0,MYF(0))) goto err; if (! (flags & HA_DONT_TOUCH_DATA)) { #ifdef USE_RELOC - if (my_chsize(dfile,share.base.min_pack_length*ci->reloc_rows,MYF(0))) + if (my_chsize(dfile,share.base.min_pack_length*ci->reloc_rows,0,MYF(0))) goto err; #endif errpos=2; @@ -599,20 +645,16 @@ err: VOID(my_close(dfile,MYF(0))); /* fall through */ case 2: - if (! (flags & HA_DONT_TOUCH_DATA)) - { /* QQ: Tõnu should add a call to my_raid_delete() here */ - VOID(fn_format(buff,name,"",MI_NAME_DEXT,2+4)); - my_delete(buff,MYF(0)); - } + if (! (flags & HA_DONT_TOUCH_DATA)) + my_delete_with_symlink(fn_format(filename,name,"",MI_NAME_DEXT,2+4), + MYF(0)); /* fall through */ case 1: VOID(my_close(file,MYF(0))); if (! (flags & HA_DONT_TOUCH_DATA)) - { - VOID(fn_format(buff,name,"",MI_NAME_IEXT,2+4)); - my_delete(buff,MYF(0)); - } + my_delete_with_symlink(fn_format(filename,name,"",MI_NAME_IEXT,2+4), + MYF(0)); } my_free((char*) rec_per_key_part, MYF(0)); DBUG_RETURN(my_errno=save_errno); /* return the fatal errno */ diff --git a/myisam/mi_dbug.c b/myisam/mi_dbug.c index eda1aafecc8..482287938c0 100644 --- a/myisam/mi_dbug.c +++ b/myisam/mi_dbug.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -162,7 +162,7 @@ my_bool check_table_is_closed(const char *name, const char *where) { MI_INFO *info=(MI_INFO*) pos->data; MYISAM_SHARE *share=info->s; - if (!strcmp(share->filename,filename)) + if (!strcmp(share->unique_file_name,filename)) { if (share->last_version) { diff --git a/myisam/mi_delete.c b/myisam/mi_delete.c index 445e745b07d..6f94e3c4256 100644 --- a/myisam/mi_delete.c +++ b/myisam/mi_delete.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -97,6 +97,12 @@ int mi_delete(MI_INFO *info,const byte *record) myisam_log_command(MI_LOG_DELETE,info,(byte*) lastpos,sizeof(lastpos),0); VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); allow_break(); /* Allow SIGHUP & SIGINT */ + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (delete)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } DBUG_RETURN(0); err: @@ -196,7 +202,8 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, DBUG_ENTER("d_search"); DBUG_DUMP("page",(byte*) anc_buff,mi_getint(anc_buff)); - flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,key_length,SEARCH_SAME, + flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key, USE_WHOLE_KEY, + SEARCH_SAME, &keypos, lastkey, &last_key); if (flag == MI_FOUND_WRONG_KEY) { diff --git a/myisam/mi_delete_all.c b/myisam/mi_delete_all.c index c3ed9455e12..45e56626d59 100644 --- a/myisam/mi_delete_all.c +++ b/myisam/mi_delete_all.c @@ -1,21 +1,21 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* Remove all rows from a MyISAM table */ -/* This only clears the status information; The files are not truncated */ +/* This clears the status information and truncates files */ #include "myisamdef.h" @@ -43,12 +43,20 @@ int mi_delete_all_rows(MI_INFO *info) info->state->empty=info->state->key_empty=0; state->checksum=0; - for (i=share->base.max_key_block_length/MI_KEY_BLOCK_LENGTH ; i-- ; ) + for (i=share->base.max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH ; i-- ; ) state->key_del[i]= HA_OFFSET_ERROR; for (i=0 ; i < share->base.keys ; i++) state->key_root[i]= HA_OFFSET_ERROR; myisam_log_command(MI_LOG_DELETE_ALL,info,(byte*) 0,0,0); + /* + If we are using delayed keys or if the user has done changes to the tables + since it was locked then there may be key blocks in the key cache + */ + flush_key_blocks(share->kfile, FLUSH_IGNORE_CHANGED); + if (my_chsize(info->dfile, 0, 0, MYF(MY_WME)) || + my_chsize(share->kfile, share->base.keystart, 0, MYF(MY_WME)) ) + goto err; VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); allow_break(); /* Allow SIGHUP & SIGINT */ DBUG_RETURN(0); @@ -62,4 +70,3 @@ err: DBUG_RETURN(my_errno=save_errno); } } /* mi_delete */ - diff --git a/myisam/mi_delete_table.c b/myisam/mi_delete_table.c index 995106160ef..6d842d7e6a4 100644 --- a/myisam/mi_delete_table.c +++ b/myisam/mi_delete_table.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -50,12 +50,12 @@ int mi_delete_table(const char *name) #endif /* USE_RAID */ fn_format(from,name,"",MI_NAME_IEXT,4); - if (my_delete(from, MYF(MY_WME))) + if (my_delete_with_symlink(from, MYF(MY_WME))) DBUG_RETURN(my_errno); fn_format(from,name,"",MI_NAME_DEXT,4); #ifdef USE_RAID if (raid_type) DBUG_RETURN(my_raid_delete(from, raid_chunks, MYF(MY_WME)) ? my_errno : 0); #endif - DBUG_RETURN(my_delete(from, MYF(MY_WME)) ? my_errno : 0); + DBUG_RETURN(my_delete_with_symlink(from, MYF(MY_WME)) ? my_errno : 0); } diff --git a/myisam/mi_dynrec.c b/myisam/mi_dynrec.c index c9fe493744d..d33aa2718b7 100644 --- a/myisam/mi_dynrec.c +++ b/myisam/mi_dynrec.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -60,15 +60,17 @@ int _mi_write_blob_record(MI_INFO *info, const byte *record) int error; ulong reclength,extra; - extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ - MI_DYN_DELETE_BLOCK_HEADER+1; - reclength=info->s->base.pack_reclength+ - _my_calc_total_blob_length(info,record)+ extra; + extra= (ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER+1); + reclength= (info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,record)+ extra); +#ifdef NOT_USED /* We now support big rows */ if (reclength > MI_DYN_MAX_ROW_LENGTH) { my_errno=HA_ERR_TO_BIG_ROW; return -1; } +#endif if (!(rec_buff=(byte*) my_alloca(reclength))) { my_errno=ENOMEM; @@ -89,15 +91,17 @@ int _mi_update_blob_record(MI_INFO *info, my_off_t pos, const byte *record) int error; ulong reclength,extra; - extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ - MI_DYN_DELETE_BLOCK_HEADER; - reclength=info->s->base.pack_reclength+ - _my_calc_total_blob_length(info,record)+ extra; + extra= (ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER); + reclength= (info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,record)+ extra); +#ifdef NOT_USED /* We now support big rows */ if (reclength > MI_DYN_MAX_ROW_LENGTH) { my_errno=HA_ERR_TO_BIG_ROW; return -1; } +#endif if (!(rec_buff=(byte*) my_alloca(reclength))) { my_errno=ENOMEM; @@ -130,14 +134,14 @@ static int write_dynamic_record(MI_INFO *info, const byte *record, DBUG_ENTER("write_dynamic_record"); flag=0; - while (reclength) + do { if (_mi_find_writepos(info,reclength,&filepos,&length)) goto err; if (_mi_write_part_record(info,filepos,length,info->s->state.dellink, (byte**) &record,&reclength,&flag)) goto err; - } + } while (reclength); DBUG_RETURN(0); err: @@ -153,6 +157,7 @@ static int _mi_find_writepos(MI_INFO *info, ulong *length) /* length of block at filepos */ { MI_BLOCK_INFO block_info; + ulong tmp; DBUG_ENTER("_mi_find_writepos"); if (info->s->state.dellink != HA_OFFSET_ERROR) @@ -178,19 +183,22 @@ static int _mi_find_writepos(MI_INFO *info, { /* No deleted blocks; Allocate a new block */ *filepos=info->state->data_file_length; - if ((*length=reclength+3 + test(reclength >= (65520-3))) < + if ((tmp=reclength+3 + test(reclength >= (65520-3))) < info->s->base.min_block_length) - *length=info->s->base.min_block_length; + tmp= info->s->base.min_block_length; else - *length= ((*length+MI_DYN_ALIGN_SIZE-1) & - (~ (ulong) (MI_DYN_ALIGN_SIZE-1))); + tmp= ((tmp+MI_DYN_ALIGN_SIZE-1) & + (~ (ulong) (MI_DYN_ALIGN_SIZE-1))); if (info->state->data_file_length > - (info->s->base.max_data_file_length- *length)) + (info->s->base.max_data_file_length - tmp)) { my_errno=HA_ERR_RECORD_FILE_FULL; DBUG_RETURN(-1); } - info->state->data_file_length+= *length; + if (tmp > MI_MAX_BLOCK_LENGTH) + tmp=MI_MAX_BLOCK_LENGTH; + *length= tmp; + info->state->data_file_length+= tmp; info->s->state.split++; info->update|=HA_STATE_WRITE_AT_END; } @@ -370,19 +378,30 @@ int _mi_write_part_record(MI_INFO *info, info->s->state.dellink : info->state->data_file_length; if (*flag == 0) /* First block */ { - head_length=5+8+long_block*2; - temp[0]=5+(uchar) long_block; - if (long_block) + if (*reclength > MI_MAX_BLOCK_LENGTH) { - mi_int3store(temp+1,*reclength); - mi_int3store(temp+4,length-head_length); - mi_sizestore((byte*) temp+7,next_filepos); + head_length= 16; + temp[0]=13; + mi_int4store(temp+1,*reclength); + mi_int3store(temp+5,length-head_length); + mi_sizestore((byte*) temp+8,next_filepos); } else { - mi_int2store(temp+1,*reclength); - mi_int2store(temp+3,length-head_length); - mi_sizestore((byte*) temp+5,next_filepos); + head_length=5+8+long_block*2; + temp[0]=5+(uchar) long_block; + if (long_block) + { + mi_int3store(temp+1,*reclength); + mi_int3store(temp+4,length-head_length); + mi_sizestore((byte*) temp+7,next_filepos); + } + else + { + mi_int2store(temp+1,*reclength); + mi_int2store(temp+3,length-head_length); + mi_sizestore((byte*) temp+5,next_filepos); + } } } else @@ -629,7 +648,7 @@ uint _mi_rec_pack(MI_INFO *info, register byte *to, register const byte *from) } blob++; } - else if (type == FIELD_SKIPP_ZERO) + else if (type == FIELD_SKIP_ZERO) { if (memcmp((byte*) from,zero_string,length) == 0) flag|=bit; @@ -638,11 +657,11 @@ uint _mi_rec_pack(MI_INFO *info, register byte *to, register const byte *from) memcpy((byte*) to,from,(size_t) length); to+=length; } } - else if (type == FIELD_SKIPP_ENDSPACE || - type == FIELD_SKIPP_PRESPACE) + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) { pos= (byte*) from; end= (byte*) from + length; - if (type == FIELD_SKIPP_ENDSPACE) + if (type == FIELD_SKIP_ENDSPACE) { /* Pack trailing spaces */ while (end > from && *(end-1) == ' ') end--; @@ -707,11 +726,11 @@ uint _mi_rec_pack(MI_INFO *info, register byte *to, register const byte *from) /* -** Check if a record was correctly packed. Used only by isamchk -** Returns 0 if record is ok. + Check if a record was correctly packed. Used only by myisamchk + Returns 0 if record is ok. */ -my_bool _mi_rec_check(MI_INFO *info,const char *record) +my_bool _mi_rec_check(MI_INFO *info,const char *record, byte *rec_buff) { uint length,new_length,flag,bit,i; char *pos,*end,*packpos,*to; @@ -719,7 +738,7 @@ my_bool _mi_rec_check(MI_INFO *info,const char *record) reg3 MI_COLUMNDEF *rec; DBUG_ENTER("_mi_rec_check"); - packpos=info->rec_buff; to= info->rec_buff+info->s->base.pack_bits; + packpos=rec_buff; to= rec_buff+info->s->base.pack_bits; rec=info->s->rec; flag= *packpos; bit=1; @@ -737,7 +756,7 @@ my_bool _mi_rec_check(MI_INFO *info,const char *record) if (blob_length) to+=length - mi_portable_sizeof_char_ptr+ blob_length; } - else if (type == FIELD_SKIPP_ZERO) + else if (type == FIELD_SKIP_ZERO) { if (memcmp((byte*) record,zero_string,length) == 0) { @@ -747,11 +766,11 @@ my_bool _mi_rec_check(MI_INFO *info,const char *record) else to+=length; } - else if (type == FIELD_SKIPP_ENDSPACE || - type == FIELD_SKIPP_PRESPACE) + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) { pos= (byte*) record; end= (byte*) record + length; - if (type == FIELD_SKIPP_ENDSPACE) + if (type == FIELD_SKIP_ENDSPACE) { /* Pack trailing spaces */ while (end > record && *(end-1) == ' ') end--; @@ -803,7 +822,7 @@ my_bool _mi_rec_check(MI_INFO *info,const char *record) to+=length; } } - if (info->packed_length != (uint) (to - info->rec_buff) + if (info->packed_length != (uint) (to - rec_buff) + test(info->s->calc_checksum) || (bit != 1 && (flag & ~(bit - 1)))) goto err; @@ -863,10 +882,10 @@ ulong _mi_rec_unpack(register MI_INFO *info, register byte *to, byte *from, } if (flag & bit) { - if (type == FIELD_BLOB || type == FIELD_SKIPP_ZERO) + if (type == FIELD_BLOB || type == FIELD_SKIP_ZERO) bzero((byte*) to,rec_length); - else if (type == FIELD_SKIPP_ENDSPACE || - type == FIELD_SKIPP_PRESPACE) + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) { if (rec->length > 255 && *from & 128) { @@ -884,7 +903,7 @@ ulong _mi_rec_unpack(register MI_INFO *info, register byte *to, byte *from, if (length >= rec_length || min_pack_length + length > (uint) (from_end - from)) goto err; - if (type == FIELD_SKIPP_ENDSPACE) + if (type == FIELD_SKIP_ENDSPACE) { memcpy(to,(byte*) from,(size_t) length); bfill((byte*) to+length,rec_length-length,' '); @@ -911,7 +930,7 @@ ulong _mi_rec_unpack(register MI_INFO *info, register byte *to, byte *from, } else { - if (type == FIELD_SKIPP_ENDSPACE || type == FIELD_SKIPP_PRESPACE) + if (type == FIELD_SKIP_ENDSPACE || type == FIELD_SKIP_PRESPACE) min_pack_length--; if (min_pack_length + rec_length > (uint) (from_end - from)) goto err; @@ -1042,7 +1061,8 @@ int _mi_read_dynamic_record(MI_INFO *info, my_off_t filepos, byte *buf) goto panic; if (info->s->base.blobs) { - if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + if (!(to=mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) goto err; } else @@ -1059,11 +1079,11 @@ int _mi_read_dynamic_record(MI_INFO *info, my_off_t filepos, byte *buf) } while (left_length); info->update|= HA_STATE_AKTIV; /* We have a aktive record */ - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); DBUG_RETURN(_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != MY_FILE_ERROR ? 0 : -1); } - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); DBUG_RETURN(-1); /* Wrong data to read */ panic: @@ -1073,33 +1093,12 @@ err: DBUG_RETURN(-1); } - -byte *mi_fix_rec_buff_for_blob(MI_INFO *info, ulong length) -{ - uint extra; - if (! info->rec_buff || length > info->alloced_rec_buff_length) - { - byte *newptr; - extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ - MI_DYN_DELETE_BLOCK_HEADER; - if (!(newptr=(byte*) my_realloc((gptr) info->rec_alloc,length+extra, - MYF(MY_ALLOW_ZERO_PTR)))) - return newptr; - info->rec_alloc=newptr; - info->rec_buff=newptr+ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER); - info->alloced_rec_buff_length=length; - } - return info->rec_buff; -} - - /* compare unique constraint between stored rows */ int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, const byte *record, my_off_t pos) { - byte *rec_buff,*rec_alloc,*old_record; - uint alloced_rec_buff_length; + byte *rec_buff,*old_record; int error; DBUG_ENTER("_mi_cmp_dynamic_unique"); @@ -1108,23 +1107,15 @@ int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, /* Don't let the compare destroy blobs that may be in use */ rec_buff=info->rec_buff; - rec_alloc=info->rec_alloc; - alloced_rec_buff_length=info->alloced_rec_buff_length; if (info->s->base.blobs) - { info->rec_buff=0; - info->rec_alloc=0; - info->alloced_rec_buff_length=0; - } error=_mi_read_dynamic_record(info,pos,old_record); if (!error) error=mi_unique_comp(def, record, old_record, def->null_are_equal); if (info->s->base.blobs) { - my_free(info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); + my_free(mi_get_rec_buff_ptr(info, info->rec_buff), MYF(MY_ALLOW_ZERO_PTR)); info->rec_buff=rec_buff; - info->rec_alloc=rec_alloc; - info->alloced_rec_buff_length=alloced_rec_buff_length; } my_afree(old_record); DBUG_RETURN(error); @@ -1258,7 +1249,7 @@ int _mi_read_rnd_dynamic_record(MI_INFO *info, byte *buf, if (info->lock_type == F_UNLCK) { #ifndef UNSAFE_LOCKING - if (share->r_locks == 0 && share->w_locks == 0) + if (share->tot_locks == 0) { if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) @@ -1334,7 +1325,8 @@ int _mi_read_rnd_dynamic_record(MI_INFO *info, byte *buf, info->lastpos=filepos; if (share->base.blobs) { - if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + if (!(to= mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) goto err; } else @@ -1393,8 +1385,7 @@ int _mi_read_rnd_dynamic_record(MI_INFO *info, byte *buf, } while (left_len); info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; - if (share->r_locks == 0 && share->w_locks == 0) - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); if (_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != MY_FILE_ERROR) DBUG_RETURN(0); @@ -1429,15 +1420,15 @@ uint _mi_get_block_info(MI_BLOCK_INFO *info, File file, my_off_t filepos) DBUG_DUMP("header",(byte*) header,MI_BLOCK_INFO_HEADER_LENGTH); if (info->second_read) { - if (info->header[0] <= 6) + if (info->header[0] <= 6 || info->header[0] == 13) return_val=BLOCK_SYNC_ERROR; } else { - if (info->header[0] > 6) + if (info->header[0] > 6 && info->header[0] != 13) return_val=BLOCK_SYNC_ERROR; } - info->next_filepos= HA_OFFSET_ERROR; /* Dummy ifall no next block */ + info->next_filepos= HA_OFFSET_ERROR; /* Dummy if no next block */ switch (info->header[0]) { case 0: @@ -1471,6 +1462,14 @@ uint _mi_get_block_info(MI_BLOCK_INFO *info, File file, my_off_t filepos) info->filepos=filepos+4; return return_val | BLOCK_FIRST | BLOCK_LAST; + case 13: + info->rec_len=mi_uint4korr(header+1); + info->block_len=info->data_len=mi_uint3korr(header+5); + info->next_filepos=mi_sizekorr(header+8); + info->second_read=1; + info->filepos=filepos+16; + return return_val | BLOCK_FIRST; + case 3: info->rec_len=info->data_len=mi_uint2korr(header+1); info->block_len=info->rec_len+ (uint) header[3]; diff --git a/myisam/mi_extra.c b/myisam/mi_extra.c index cf075512ac4..75057dd4e6a 100644 --- a/myisam/mi_extra.c +++ b/myisam/mi_extra.c @@ -1,24 +1,19 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Extra functions we want to do with a database */ -/* - Set flags for quicker databasehandler */ -/* - Set databasehandler to normal */ -/* - Reset recordpointers as after open database */ - #include "myisamdef.h" #ifdef HAVE_MMAP #include <sys/mman.h> @@ -27,30 +22,55 @@ #include <errno.h> #endif - /* set extra flags for database */ +/* + Set options and buffers to optimize table handling -int mi_extra(MI_INFO *info, enum ha_extra_function function) + SYNOPSIS + mi_extra() + info open table + function operation + extra_arg Pointer to extra argument (normally pointer to ulong) + Used when function is one of: + HA_EXTRA_WRITE_CACHE + HA_EXTRA_CACHE + HA_EXTRA_BULK_INSERT_BEGIN + If extra_arg is 0, then the default cache size is used. + HA_EXTRA_BULK_INSERT_FLUSH + extra_arg is a a pointer to which index to flush (uint*) + RETURN VALUES + 0 ok +*/ + + +int mi_extra(MI_INFO *info, enum ha_extra_function function, void *extra_arg) { int error=0; + ulong cache_size; MYISAM_SHARE *share=info->s; DBUG_ENTER("mi_extra"); + DBUG_PRINT("enter",("function: %d",(int) function)); switch (function) { case HA_EXTRA_RESET: /* Free buffers and reset the following flags: EXTRA_CACHE, EXTRA_WRITE_CACHE, EXTRA_KEYREAD, EXTRA_QUICK + + If the row buffer cache is large (for dynamic tables), reduce it + to save memory. */ if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) { info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); error=end_io_cache(&info->rec_cache); } + if (share->base.blobs) + mi_alloc_rec_buff(info, -1, &info->rec_buff); #if defined(HAVE_MMAP) && defined(HAVE_MADVICE) if (info->opt_flag & MEMMAP_USED) madvise(share->file_map,share->state.state.data_file_length,MADV_RANDOM); #endif - info->opt_flag&= ~(KEY_READ_USED | REMEMBER_OLD_POS); + info->opt_flag&= ~(KEY_READ_USED | REMEMBER_OLD_POS); info->quick_mode=0; /* Fall through */ @@ -102,11 +122,13 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) if (!(info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED | MEMMAP_USED))) { + cache_size= (extra_arg ? *(ulong*) extra_arg : + my_default_record_cache_size); if (!(init_io_cache(&info->rec_cache,info->dfile, (uint) min(info->state->data_file_length+1, - my_default_record_cache_size), - READ_CACHE,0L,(pbool) (info->lock_type != F_UNLCK), - MYF(share->write_flag & MY_WAIT_IF_FULL)))) + cache_size), + READ_CACHE,0L,(pbool) (info->lock_type != F_UNLCK), + MYF(share->write_flag & MY_WAIT_IF_FULL)))) { info->opt_flag|=READ_CACHE_USED; info->update&= ~HA_STATE_ROW_CHANGED; @@ -132,10 +154,12 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) error=1; /* Not possibly if not locked */ break; } + cache_size= (extra_arg ? *(ulong*) extra_arg : + my_default_record_cache_size); if (!(info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED | OPT_NO_ROWS)) && !share->state.header.uniques) - if (!(init_io_cache(&info->rec_cache,info->dfile,0, + if (!(init_io_cache(&info->rec_cache,info->dfile, cache_size, WRITE_CACHE,info->state->data_file_length, (pbool) (info->lock_type != F_UNLCK), MYF(share->write_flag & MY_WAIT_IF_FULL)))) @@ -146,6 +170,10 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) HA_STATE_EXTEND_BLOCK); } break; + case HA_EXTRA_PREPARE_FOR_UPDATE: + if (info->s->data_file_type != DYNAMIC_RECORD) + break; + /* Remove read/write cache if dynamic rows */ case HA_EXTRA_NO_CACHE: if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) { @@ -219,9 +247,17 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) } if (share->state.key_map) { - share->state.key_map=0; - info->state->key_file_length=share->state.state.key_file_length= - share->base.keystart; + MI_KEYDEF *key=share->keyinfo; + uint i; + for (i=0 ; i < share->base.keys ; i++,key++) + { + if (!(key->flag & HA_NOSAME) && info->s->base.auto_key != i+1) + { + share->state.key_map&= ~ ((ulonglong) 1 << i); + info->update|= HA_STATE_CHANGED; + } + } + if (!share->changed) { share->state.changed|= STATE_CHANGED | STATE_NOT_ANALYZED; @@ -237,12 +273,15 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) } break; case HA_EXTRA_FORCE_REOPEN: + case HA_EXTRA_PREPARE_FOR_DELETE: pthread_mutex_lock(&THR_LOCK_myisam); share->last_version= 0L; /* Impossible version */ #ifdef __WIN__ /* Close the isam and data files as Win32 can't drop an open table */ pthread_mutex_lock(&share->intern_lock); - if (flush_key_blocks(share->kfile,FLUSH_RELEASE)) + if (flush_key_blocks(share->kfile, + (function == HA_EXTRA_FORCE_REOPEN ? + FLUSH_RELEASE : FLUSH_IGNORE_CHANGED))) { error=my_errno; share->changed=1; @@ -316,11 +355,7 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function) } } if (share->base.blobs) - { - my_free(info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); - info->rec_alloc=info->rec_buff=0; - mi_fix_rec_buff_for_blob(info,info->s->base.pack_reclength); - } + mi_alloc_rec_buff(info, -1, &info->rec_buff); break; case HA_EXTRA_NORMAL: /* Theese isn't in use */ info->quick_mode=0; diff --git a/myisam/mi_info.c b/myisam/mi_info.c index 6e7abfc0914..0be3cc44d80 100644 --- a/myisam/mi_info.c +++ b/myisam/mi_info.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -45,7 +45,7 @@ int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag) { pthread_mutex_lock(&share->intern_lock); VOID(_mi_readinfo(info,F_RDLCK,0)); - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); pthread_mutex_unlock(&share->intern_lock); } if (flag & HA_STATUS_VARIABLE) @@ -87,6 +87,8 @@ int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag) x->raid_chunks= share->base.raid_chunks; x->raid_chunksize= share->base.raid_chunksize; x->key_map = share->state.key_map; + x->data_file_name = share->data_file_name; + x->index_file_name = share->index_file_name; } if ((flag & HA_STATUS_TIME) && !my_fstat(info->dfile,&state,MYF(0))) x->update_time=state.st_mtime; diff --git a/myisam/mi_key.c b/myisam/mi_key.c index 9f4e2cb1524..9ec1ca99e0e 100644 --- a/myisam/mi_key.c +++ b/myisam/mi_key.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -18,6 +18,9 @@ #include "myisamdef.h" #include "m_ctype.h" +#ifdef HAVE_IEEEFP_H +#include <ieeefp.h> +#endif #define CHECK_KEYS @@ -88,25 +91,28 @@ uint _mi_make_key(register MI_INFO *info, uint keynr, uchar *key, } else if (keyseg->flag & HA_SWAP_KEY) { /* Numerical column */ -#ifdef NAN_TEST - float float_nr; - double dbl_nr; +#ifdef HAVE_ISNAN if (type == HA_KEYTYPE_FLOAT) { - float_nr=float4get(pos); - if (float_nr == (float) FLT_MAX) + float nr; + float4get(nr,pos); + if (isnan(nr)) { - float_nr= (float) FLT_MAX; - pos= (byte*) &float_nr; + /* Replace NAN with zero */ + bzero(key,length); + key+=length; + continue; } } else if (type == HA_KEYTYPE_DOUBLE) { - dbl_nr=float8get(key); - if (dbl_nr == DBL_MAX) + double nr; + float8get(nr,pos); + if (isnan(nr)) { - dbl_nr=DBL_MAX; - pos=(byte*) &dbl_nr; + bzero(key,length); + key+=length; + continue; } } #endif @@ -244,7 +250,8 @@ static int _mi_put_key_in_record(register MI_INFO *info, uint keynr, if (info->blobs && info->s->keyinfo[keynr].flag & HA_VAR_LENGTH_KEY) { if (!(blob_ptr= - mi_fix_rec_buff_for_blob(info, info->s->keyinfo[keynr].keylength))) + mi_alloc_rec_buff(info, info->s->keyinfo[keynr].keylength, + &info->rec_buff))) goto err; } key=(byte*) info->lastkey; @@ -346,7 +353,7 @@ err: int _mi_read_key_record(MI_INFO *info, my_off_t filepos, byte *buf) { - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); if (filepos != HA_OFFSET_ERROR) { if (info->lastinx >= 0) diff --git a/myisam/mi_locking.c b/myisam/mi_locking.c index de5f0d0c2cf..cbde05d31f5 100644 --- a/myisam/mi_locking.c +++ b/myisam/mi_locking.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -43,13 +43,13 @@ int mi_lock_database(MI_INFO *info, int lock_type) pthread_mutex_lock(&share->intern_lock); if (share->kfile >= 0) /* May only be false on windows */ { - switch (lock_type) - { + switch (lock_type) { case F_UNLCK: if (info->lock_type == F_RDLCK) count= --share->r_locks; else count= --share->w_locks; + --share->tot_locks; if (info->lock_type == F_WRLCK && !share->w_locks && !share->delay_key_write && flush_key_blocks(share->kfile,FLUSH_KEEP)) { @@ -153,6 +153,7 @@ int mi_lock_database(MI_INFO *info, int lock_type) } VOID(_mi_test_if_changed(info)); share->r_locks++; + share->tot_locks++; info->lock_type=lock_type; break; case F_WRLCK: @@ -199,7 +200,9 @@ int mi_lock_database(MI_INFO *info, int lock_type) } VOID(_mi_test_if_changed(info)); info->lock_type=lock_type; + info->invalidator=info->s->invalidator; share->w_locks++; + share->tot_locks++; break; default: break; /* Impossible */ @@ -216,7 +219,7 @@ int mi_lock_database(MI_INFO *info, int lock_type) /**************************************************************************** -** The following functions are called by thr_lock() in threaded applications + The following functions are called by thr_lock() in threaded applications ****************************************************************************/ void mi_get_status(void* param) @@ -295,13 +298,12 @@ my_bool mi_check_status(void* param) int _mi_readinfo(register MI_INFO *info, int lock_type, int check_keybuffer) { - MYISAM_SHARE *share; DBUG_ENTER("_mi_readinfo"); - share=info->s; if (info->lock_type == F_UNLCK) { - if (!share->r_locks && !share->w_locks) + MYISAM_SHARE *share=info->s; + if (!share->tot_locks) { if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, info->lock_wait | MY_SEEK_NOT_DONE)) @@ -317,6 +319,7 @@ int _mi_readinfo(register MI_INFO *info, int lock_type, int check_keybuffer) } if (check_keybuffer) VOID(_mi_test_if_changed(info)); + info->invalidator=info->s->invalidator; } else if (lock_type == F_WRLCK && info->lock_type == F_RDLCK) { @@ -338,7 +341,7 @@ int _mi_writeinfo(register MI_INFO *info, uint operation) DBUG_ENTER("_mi_writeinfo"); error=0; - if (share->r_locks == 0 && share->w_locks == 0) + if (share->tot_locks == 0) { olderror=my_errno; /* Remember last error */ if (operation) @@ -411,11 +414,14 @@ int _mi_mark_file_changed(MI_INFO *info) share->global_changed=1; share->state.open_count++; } - mi_int2store(buff,share->state.open_count); - buff[2]=1; /* Mark that it's changed */ - return (my_pwrite(share->kfile,buff,sizeof(buff), - sizeof(share->state.header), - MYF(MY_NABP))); + if (!share->temporary) + { + mi_int2store(buff,share->state.open_count); + buff[2]=1; /* Mark that it's changed */ + return (my_pwrite(share->kfile,buff,sizeof(buff), + sizeof(share->state.header), + MYF(MY_NABP))); + } } return 0; } diff --git a/myisam/mi_log.c b/myisam/mi_log.c index 5c64c130212..1dcfd5250d2 100644 --- a/myisam/mi_log.c +++ b/myisam/mi_log.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_open.c b/myisam/mi_open.c index 1832d525157..60049325c5c 100644 --- a/myisam/mi_open.c +++ b/myisam/mi_open.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -56,7 +56,7 @@ static MI_INFO *test_if_reopen(char *filename) { MI_INFO *info=(MI_INFO*) pos->data; MYISAM_SHARE *share=info->s; - if (!strcmp(share->filename,filename) && share->last_version) + if (!strcmp(share->unique_file_name,filename) && share->last_version) return info; } return 0; @@ -74,9 +74,11 @@ static MI_INFO *test_if_reopen(char *filename) MI_INFO *mi_open(const char *name, int mode, uint open_flags) { int lock_error,kfile,open_mode,save_errno; - uint i,j,len,errpos,head_length,base_pos,offset,info_length,extra,keys, + uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys, key_parts,unique_key_parts,tmp_length,uniques; - char name_buff[FN_REFLEN],*disk_cache,*disk_pos, *end_pos; + char name_buff[FN_REFLEN], org_name [FN_REFLEN], index_name[FN_REFLEN], + data_name[FN_REFLEN]; + char *disk_cache, *disk_pos, *end_pos; MI_INFO info,*m_info,*old_info; MYISAM_SHARE share_buff,*share; ulong rec_per_key_part[MI_MAX_POSSIBLE_KEY*MI_MAX_KEY_SEG]; @@ -91,7 +93,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) head_length=sizeof(share_buff.state.header); bzero((byte*) &info,sizeof(info)); - VOID(fn_format(name_buff,name,"",MI_NAME_IEXT,4+16+32)); + my_realpath(name_buff, fn_format(org_name,name,"",MI_NAME_IEXT,4),MYF(0)); pthread_mutex_lock(&THR_LOCK_myisam); if (!(old_info=test_if_reopen(name_buff))) { @@ -120,7 +122,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) DBUG_PRINT("error",("Wrong header in %s",name_buff)); DBUG_DUMP("error_dump",(char*) share->state.header.file_version, head_length); - my_errno=HA_ERR_CRASHED; + my_errno=HA_ERR_NOT_A_TABLE; goto err; } share->options= mi_uint2korr(share->state.header.options); @@ -135,6 +137,13 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) my_errno=HA_ERR_OLD_FILE; goto err; } + /* Don't call realpath() if the name can't be a link */ + if (strcmp(name_buff, org_name)) + (void) my_readlink(index_name, org_name, MYF(0)); + else + (void) strmov(index_name, org_name); + (void) fn_format(data_name,org_name,"",MI_NAME_DEXT,2+4+16); + info_length=mi_uint2korr(share->state.header.header_length); base_pos=mi_uint2korr(share->state.header.base_pos); if (!(disk_cache=(char*) my_alloca(info_length+128))) @@ -216,7 +225,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) (mi_safe_mul(share->base.pack_reclength, (ulonglong) 1 << (share->base.rec_reflength*8))-1); max_key_file_length= - mi_safe_mul(MI_KEY_BLOCK_LENGTH, + mi_safe_mul(MI_MIN_KEY_BLOCK_LENGTH, ((ulonglong) 1 << (share->base.key_reflength*8))-1); #if SIZEOF_OFF_T == 4 set_if_smaller(max_data_file_length, INT_MAX32); @@ -258,7 +267,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) &share->rec, (share->base.fields+1)*sizeof(MI_COLUMNDEF), &share->blobs,sizeof(MI_BLOB)*share->base.blobs, - &share->filename,strlen(name_buff)+1, + &share->unique_file_name,strlen(name_buff)+1, + &share->index_file_name,strlen(index_name)+1, + &share->data_file_name,strlen(data_name)+1, &share->state.key_root,keys*sizeof(my_off_t), &share->state.key_del, (share->state.header.max_block_size*sizeof(my_off_t)), @@ -276,7 +287,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) memcpy((char*) share->state.key_del, (char*) key_del, (sizeof(my_off_t) * share->state.header.max_block_size)); - strmov(share->filename,name_buff); + strmov(share->unique_file_name, name_buff); + strmov(share->index_file_name, index_name); + strmov(share->data_file_name, data_name); share->blocksize=min(IO_SIZE,myisam_block_size); { @@ -368,7 +381,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) lock_error=1; /* Database unlocked */ } - if (mi_open_datafile(&info, share)) + if (mi_open_datafile(&info, share, -1)) goto err; errpos=5; @@ -441,7 +454,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) my_errno=EACCES; /* Can't open in write mode */ goto err; } - if (mi_open_datafile(&info, share)) + if (mi_open_datafile(&info, share, old_info->dfile)) goto err; errpos=5; } @@ -453,12 +466,12 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) &info.buff,(share->base.max_key_block_length*2+ share->base.max_key_length), &info.lastkey,share->base.max_key_length*3+1, - &info.filename,strlen(name)+1, + &info.filename,strlen(org_name)+1, NullS)) goto err; errpos=6; - strmov(info.filename,name); + strmov(info.filename,org_name); memcpy(info.blobs,share->blobs,sizeof(MI_BLOB)*share->base.blobs); info.lastkey2=info.lastkey+share->base.max_key_length; @@ -476,6 +489,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) share->options|=HA_OPTION_READ_ONLY_DATA; info.lock_type=F_UNLCK; info.quick_mode=0; + info.bulk_insert=0; info.errkey= -1; info.page_changed=1; pthread_mutex_lock(&share->intern_lock); @@ -486,6 +500,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) { info.lock_type=F_RDLCK; share->r_locks++; + share->tot_locks++; } if ((open_flags & HA_OPEN_TMP_TABLE) || (share->options & HA_OPTION_TMP_TABLE)) @@ -493,6 +508,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) share->temporary=share->delay_key_write=1; share->write_flag=MYF(MY_NABP); share->w_locks++; /* We don't have to update status */ + share->tot_locks++; info.lock_type=F_WRLCK; } if (((open_flags & HA_OPEN_DELAY_KEY_WRITE) || @@ -504,21 +520,10 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) /* Allocate buffer for one record */ - extra=0; - if (share->options & HA_OPTION_PACK_RECORD) - extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ - MI_DYN_DELETE_BLOCK_HEADER; - - tmp_length=max(share->base.pack_reclength+share->base.pack_bits, - share->base.max_key_length); - info.alloced_rec_buff_length=tmp_length; - if (!(info.rec_alloc=(byte*) my_malloc(tmp_length+extra+8, - MYF(MY_WME | MY_ZEROFILL)))) + /* prerequisites: bzero(info) && info->s=share; are met. */ + if (!mi_alloc_rec_buff(&info, -1, &info.rec_buff)) goto err; - if (extra) - info.rec_buff=info.rec_alloc+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER); - else - info.rec_buff=info.rec_alloc; + bzero(info.rec_buff, mi_get_rec_buff_len(&info, info.rec_buff)); *m_info=info; #ifdef THREAD @@ -530,7 +535,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) pthread_mutex_unlock(&THR_LOCK_myisam); if (myisam_log_file >= 0) { - intern_filename(name_buff,share->filename); + intern_filename(name_buff,share->index_file_name); _myisam_log(MI_LOG_OPEN,m_info,name_buff,(uint) strlen(name_buff)); } DBUG_RETURN(m_info); @@ -569,6 +574,41 @@ err: } /* mi_open */ +byte *mi_alloc_rec_buff(MI_INFO *info, ulong length, byte **buf) +{ + uint extra; + uint32 old_length; + LINT_INIT(old_length); + + if (! *buf || length > (old_length=mi_get_rec_buff_len(info, *buf))) + { + byte *newptr = *buf; + + /* to simplify initial init of info->rec_buf in mi_open and mi_extra */ + if (length == (ulong) -1) + { + length= max(info->s->base.pack_reclength+info->s->base.pack_bits, + info->s->base.max_key_length); + /* Avoid unnecessary realloc */ + if (newptr && length == old_length) + return newptr; + } + + extra= ((info->s->options & HA_OPTION_PACK_RECORD) ? + ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_REC_BUFF_OFFSET : 0); + if (extra && newptr) + newptr-= MI_REC_BUFF_OFFSET; + if (!(newptr=(byte*) my_realloc((gptr)newptr, length+extra+8, + MYF(MY_ALLOW_ZERO_PTR)))) + return newptr; + *((uint32 *) newptr)= (uint32) length; + *buf= newptr+(extra ? MI_REC_BUFF_OFFSET : 0); + } + return *buf; +} + + ulonglong mi_safe_mul(ulonglong a, ulonglong b) { ulonglong max_val= ~ (ulonglong) 0; /* my_off_t is unsigned */ @@ -641,15 +681,20 @@ static void setup_key_functions(register MI_KEYDEF *keyinfo) } else if (keyinfo->flag & HA_VAR_LENGTH_KEY) { - keyinfo->bin_search=_mi_seq_search; keyinfo->get_key= _mi_get_pack_key; if (keyinfo->seg[0].flag & HA_PACK_KEY) { /* Prefix compression */ + if (!keyinfo->seg->charset || use_strcoll(keyinfo->seg->charset) || + (keyinfo->seg->flag & HA_NULL_PART)) + keyinfo->bin_search=_mi_seq_search; + else + keyinfo->bin_search=_mi_prefix_search; keyinfo->pack_key=_mi_calc_var_pack_key_length; keyinfo->store_key=_mi_store_var_pack_key; } else { + keyinfo->bin_search=_mi_seq_search; keyinfo->pack_key=_mi_calc_var_key_length; /* Variable length key */ keyinfo->store_key=_mi_store_static_key; } @@ -788,14 +833,17 @@ uint mi_state_info_read_dsk(File file, MI_STATE_INFO *state, my_bool pRead) { char buff[MI_STATE_INFO_SIZE + MI_STATE_EXTRA_SIZE]; - if (pRead) + if (!myisam_single_user) { - if (my_pread(file, buff, state->state_length,0L, MYF(MY_NABP))) + if (pRead) + { + if (my_pread(file, buff, state->state_length,0L, MYF(MY_NABP))) + return (MY_FILE_ERROR); + } + else if (my_read(file, buff, state->state_length,MYF(MY_NABP))) return (MY_FILE_ERROR); + mi_state_info_read(buff, state); } - else if (my_read(file, buff, state->state_length,MYF(MY_NABP))) - return (MY_FILE_ERROR); - mi_state_info_read(buff, state); return 0; } @@ -907,7 +955,7 @@ char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef) keydef->keylength = mi_uint2korr(ptr); ptr +=2; keydef->minlength = mi_uint2korr(ptr); ptr +=2; keydef->maxlength = mi_uint2korr(ptr); ptr +=2; - keydef->block_size = keydef->block_length/MI_KEY_BLOCK_LENGTH-1; + keydef->block_size = keydef->block_length/MI_MIN_KEY_BLOCK_LENGTH-1; keydef->underflow_block_length=keydef->block_length/3; keydef->version = 0; /* Not saved */ return ptr; @@ -1003,37 +1051,37 @@ char *mi_recinfo_read(char *ptr, MI_COLUMNDEF *recinfo) } /************************************************************************** - ** Help functions for recover - *************************************************************************/ +Open data file with or without RAID +We can't use dup() here as the data file descriptors need to have different +active seek-positions. -int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) -{ - char name_buff[FN_REFLEN]; - (void) fn_format(name_buff, share->filename,"",MI_NAME_DEXT, 2+4); +The argument file_to_dup is here for the future if there would on some OS +exist a dup()-like call that would give us two different file descriptors. +*************************************************************************/ +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share, File file_to_dup __attribute__((unused))) +{ #ifdef USE_RAID if (share->base.raid_type) { - if ((info->dfile=my_raid_open(name_buff, - share->mode | O_SHARE, - share->base.raid_type, - share->base.raid_chunks, - share->base.raid_chunksize, - MYF(MY_WME | MY_RAID))) < 0) - return 1; + info->dfile=my_raid_open(share->data_file_name, + share->mode | O_SHARE, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(MY_WME | MY_RAID)); } else #endif - if ((info->dfile=my_open(name_buff, share->mode | O_SHARE, - MYF(MY_WME))) < 0) - return 1; - return 0; + info->dfile=my_open(share->data_file_name, share->mode | O_SHARE, + MYF(MY_WME)); + return info->dfile >= 0 ? 0 : 1; } int mi_open_keyfile(MYISAM_SHARE *share) { - if ((share->kfile=my_open(share->filename, share->mode | O_SHARE, + if ((share->kfile=my_open(share->unique_file_name, share->mode | O_SHARE, MYF(MY_WME))) < 0) return 1; return 0; diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c index 242883a2c4a..66cfd169026 100644 --- a/myisam/mi_packrec.c +++ b/myisam/mi_packrec.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -416,8 +416,7 @@ int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, byte *buf) DBUG_RETURN(-1); /* _search() didn't find record */ file=info->dfile; - if (_mi_pack_get_block_info(info, &block_info, file, filepos, - info->rec_buff)) + if (_mi_pack_get_block_info(info, &block_info, file, filepos)) goto err; if (my_read(file,(byte*) info->rec_buff + block_info.offset , block_info.rec_len - block_info.offset, MYF(MY_NABP))) @@ -465,7 +464,7 @@ static void (*get_unpack_function(MI_COLUMNDEF *rec)) (MI_COLUMNDEF *, MI_BIT_BUFF *, uchar *, uchar *) { switch (rec->base_type) { - case FIELD_SKIPP_ZERO: + case FIELD_SKIP_ZERO: if (rec->pack_type & PACK_TYPE_ZERO_FILL) return &uf_zerofill_skipp_zero; return &uf_skipp_zero; @@ -475,7 +474,7 @@ static void (*get_unpack_function(MI_COLUMNDEF *rec)) if (rec->pack_type & PACK_TYPE_ZERO_FILL) return &uf_zerofill_normal; return &decode_bytes; - case FIELD_SKIPP_ENDSPACE: + case FIELD_SKIP_ENDSPACE: if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) { if (rec->pack_type & PACK_TYPE_SELECTED) @@ -485,7 +484,7 @@ static void (*get_unpack_function(MI_COLUMNDEF *rec)) if (rec->pack_type & PACK_TYPE_SELECTED) return &uf_endspace_selected; return &uf_endspace; - case FIELD_SKIPP_PRESPACE: + case FIELD_SKIP_PRESPACE: if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) { if (rec->pack_type & PACK_TYPE_SELECTED) @@ -963,11 +962,10 @@ int _mi_read_rnd_pack_record(MI_INFO *info, byte *buf, if (_mi_read_cache(&info->rec_cache,(byte*) block_info.header,filepos, share->pack.ref_length, skip_deleted_blocks)) goto err; - b_type=_mi_pack_get_block_info(info,&block_info,-1, filepos, NullS); + b_type=_mi_pack_get_block_info(info,&block_info,-1, filepos); } else - b_type=_mi_pack_get_block_info(info,&block_info,info->dfile,filepos, - info->rec_buff); + b_type=_mi_pack_get_block_info(info,&block_info,info->dfile,filepos); if (b_type) goto err; /* Error code is already set */ #ifndef DBUG_OFF @@ -1007,7 +1005,7 @@ int _mi_read_rnd_pack_record(MI_INFO *info, byte *buf, /* Read and process header from a huff-record-file */ uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BLOCK_INFO *info, File file, - my_off_t filepos, char *rec_buff) + my_off_t filepos) { uchar *header=info->header; uint head_length,ref_length; @@ -1057,7 +1055,8 @@ uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BLOCK_INFO *info, File file, info->blob_len=uint3korr(header+head_length+1); head_length+=4; } - if (!(mi_fix_rec_buff_for_blob(myisam,info->rec_len + info->blob_len))) + if (!(mi_alloc_rec_buff(myisam,info->rec_len + info->blob_len, + &myisam->rec_buff))) return BLOCK_FATAL_ERROR; /* not enough memory */ myisam->bit_buff.blob_pos=(uchar*) myisam->rec_buff+info->rec_len; myisam->blob_length=info->blob_len; @@ -1066,7 +1065,7 @@ uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BLOCK_INFO *info, File file, if (file > 0) { info->offset=min(info->rec_len, ref_length - head_length); - memcpy(rec_buff, header+head_length, info->offset); + memcpy(myisam->rec_buff, header+head_length, info->offset); } return 0; } @@ -1231,8 +1230,9 @@ static uchar *_mi_mempack_get_block_info(MI_INFO *myisam,MI_BLOCK_INFO *info, info->blob_len=uint3korr(header+1); header+=4; } - /* mi_fix_rec_buff_for_blob sets my_errno on error */ - if (!(mi_fix_rec_buff_for_blob(myisam,info->blob_len))) + /* mi_alloc_rec_buff sets my_errno on error */ + if (!(mi_alloc_rec_buff(myisam, info->blob_len, + &myisam->rec_buff))) return 0; /* not enough memory */ myisam->bit_buff.blob_pos=(uchar*) myisam->rec_buff; } diff --git a/myisam/mi_page.c b/myisam/mi_page.c index f8e2a977754..1d40980e309 100644 --- a/myisam/mi_page.c +++ b/myisam/mi_page.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -64,9 +64,11 @@ int _mi_write_keypage(register MI_INFO *info, register MI_KEYDEF *keyinfo, #ifndef FAST /* Safety check */ if (page < info->s->base.keystart || page+keyinfo->block_length > info->state->key_file_length || - page & (myisam_block_size-1)) + (page & (MI_MIN_KEY_BLOCK_LENGTH-1))) { - DBUG_PRINT("error",("Trying to write outside key region: %lu", + DBUG_PRINT("error",("Trying to write inside key status region: key_start: %lu length: %lu page: %lu", + (long) info->s->base.keystart, + (long) info->state->key_file_length, (long) page)); my_errno=EINVAL; return(-1); diff --git a/myisam/mi_panic.c b/myisam/mi_panic.c index 92fc6f3695c..bd0b07b097e 100644 --- a/myisam/mi_panic.c +++ b/myisam/mi_panic.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_range.c b/myisam/mi_range.c index 038f9abc3a6..70694bf4620 100644 --- a/myisam/mi_range.c +++ b/myisam/mi_range.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -45,7 +45,7 @@ ha_rows mi_records_in_range(MI_INFO *info, int inx, const byte *start_key, if ((inx = _mi_check_index(info,inx)) < 0) DBUG_RETURN(HA_POS_ERROR); - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(HA_POS_ERROR); info->update&= (HA_STATE_CHANGED+HA_STATE_ROW_CHANGED); if (info->s->concurrent_insert) @@ -58,7 +58,7 @@ ha_rows mi_records_in_range(MI_INFO *info, int inx, const byte *start_key, info->state->records+ (ha_rows) 1); if (info->s->concurrent_insert) rw_unlock(&info->s->key_root_lock[inx]); - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); if (start_pos == HA_POS_ERROR || end_pos == HA_POS_ERROR) DBUG_RETURN(HA_POS_ERROR); DBUG_PRINT("info",("records: %ld",(ulong) (end_pos-start_pos))); @@ -72,7 +72,7 @@ ha_rows mi_records_in_range(MI_INFO *info, int inx, const byte *start_key, static ha_rows _mi_record_pos(MI_INFO *info, const byte *key, uint key_len, enum ha_rkey_function search_flag) { - uint inx=(uint) info->lastinx; + uint inx=(uint) info->lastinx, nextflag; MI_KEYDEF *keyinfo=info->s->keyinfo+inx; uchar *key_buff; double pos; @@ -86,8 +86,12 @@ static ha_rows _mi_record_pos(MI_INFO *info, const byte *key, uint key_len, key_len=_mi_pack_key(info,inx,key_buff,(uchar*) key,key_len); DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg, (uchar*) key_buff,key_len);); + nextflag=myisam_read_vec[search_flag]; + if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) + key_len=USE_WHOLE_KEY; + pos=_mi_search_pos(info,keyinfo,key_buff,key_len, - myisam_read_vec[search_flag] | SEARCH_SAVE_BUFF, + nextflag | SEARCH_SAVE_BUFF, info->s->state.key_root[inx]); if (pos >= 0.0) { @@ -145,9 +149,9 @@ static double _mi_search_pos(register MI_INFO *info, ** Matches keynr+1 */ offset=1.0; /* Matches keynr+1 */ - if (nextflag & SEARCH_FIND && + if ((nextflag & SEARCH_FIND) && nod_flag && ((keyinfo->flag & (HA_NOSAME | HA_NULL_PART)) != HA_NOSAME || - key_len) && nod_flag) + key_len != USE_WHOLE_KEY)) { /* ** There may be identical keys in the tree. Try to match on of those. diff --git a/myisam/mi_rename.c b/myisam/mi_rename.c index 5c92db3f7ce..db44b8fe28f 100644 --- a/myisam/mi_rename.c +++ b/myisam/mi_rename.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -51,7 +51,7 @@ int mi_rename(const char *old_name, const char *new_name) fn_format(from,old_name,"",MI_NAME_IEXT,4); fn_format(to,new_name,"",MI_NAME_IEXT,4); - if (my_rename(from, to, MYF(MY_WME))) + if (my_rename_with_symlink(from, to, MYF(MY_WME))) DBUG_RETURN(my_errno); fn_format(from,old_name,"",MI_NAME_DEXT,4); fn_format(to,new_name,"",MI_NAME_DEXT,4); @@ -60,5 +60,5 @@ int mi_rename(const char *old_name, const char *new_name) DBUG_RETURN(my_raid_rename(from, to, raid_chunks, MYF(MY_WME)) ? my_errno : 0); #endif - DBUG_RETURN(my_rename(from, to,MYF(MY_WME)) ? my_errno : 0); + DBUG_RETURN(my_rename_with_symlink(from, to,MYF(MY_WME)) ? my_errno : 0); } diff --git a/myisam/mi_rfirst.c b/myisam/mi_rfirst.c index 8928c6332c5..e30f61801a0 100644 --- a/myisam/mi_rfirst.c +++ b/myisam/mi_rfirst.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_rkey.c b/myisam/mi_rkey.c index 0df390412b9..86547d3ef04 100644 --- a/myisam/mi_rkey.c +++ b/myisam/mi_rkey.c @@ -27,7 +27,7 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len, { uchar *key_buff; MYISAM_SHARE *share=info->s; - uint pack_key_length; + uint pack_key_length, use_key_length, nextflag; DBUG_ENTER("mi_rkey"); DBUG_PRINT("enter",("base: %lx inx: %d search_flag: %d", info,inx,search_flag)); @@ -55,11 +55,17 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len, bmove(key_buff,key,key_len); } - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) goto err; if (share->concurrent_insert) rw_rdlock(&share->key_root_lock[inx]); - if (!_mi_search(info,info->s->keyinfo+inx,key_buff,pack_key_length, + + nextflag=myisam_read_vec[search_flag]; + use_key_length=pack_key_length; + if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) + use_key_length=USE_WHOLE_KEY; + + if (!_mi_search(info,info->s->keyinfo+inx,key_buff,use_key_length, myisam_read_vec[search_flag],info->s->state.key_root[inx])) { while (info->lastpos >= info->state->data_file_length) diff --git a/myisam/mi_rlast.c b/myisam/mi_rlast.c index c08174e9117..61c3ff58fd5 100644 --- a/myisam/mi_rlast.c +++ b/myisam/mi_rlast.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_rnext.c b/myisam/mi_rnext.c index f297740af60..6d135462f96 100644 --- a/myisam/mi_rnext.c +++ b/myisam/mi_rnext.c @@ -35,7 +35,7 @@ int mi_rnext(MI_INFO *info, byte *buf, int inx) if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_PREV_FOUND) flag=0; /* Read first */ - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(my_errno); if (info->s->concurrent_insert) rw_rdlock(&info->s->key_root_lock[inx]); @@ -51,7 +51,7 @@ int mi_rnext(MI_INFO *info, byte *buf, int inx) info->s->state.key_root[inx]); else error=_mi_search(info,info->s->keyinfo+inx,info->lastkey, - info->lastkey_length,flag, info->s->state.key_root[inx]); + USE_WHOLE_KEY,flag, info->s->state.key_root[inx]); if (!error) { diff --git a/myisam/mi_rnext_same.c b/myisam/mi_rnext_same.c index 0e172894cdf..a9d1953323c 100644 --- a/myisam/mi_rnext_same.c +++ b/myisam/mi_rnext_same.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -27,15 +27,14 @@ int mi_rnext_same(MI_INFO *info, byte *buf) { int error; - uint inx,flag,not_used; + uint inx,not_used; MI_KEYDEF *keyinfo; DBUG_ENTER("mi_rnext_same"); if ((int) (inx=info->lastinx) < 0 || info->lastpos == HA_OFFSET_ERROR) DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); keyinfo=info->s->keyinfo+inx; - flag=SEARCH_BIGGER; /* Read next */ - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(my_errno); memcpy(info->lastkey2,info->lastkey,info->last_rkey_length); @@ -44,7 +43,7 @@ int mi_rnext_same(MI_INFO *info, byte *buf) for (;;) { if ((error=_mi_search_next(info,keyinfo,info->lastkey, - info->lastkey_length,flag, + info->lastkey_length,SEARCH_BIGGER, info->s->state.key_root[inx]))) break; if (_mi_key_cmp(keyinfo->seg,info->lastkey2,info->lastkey, diff --git a/myisam/mi_rprev.c b/myisam/mi_rprev.c index fff2d2257b6..4807e636252 100644 --- a/myisam/mi_rprev.c +++ b/myisam/mi_rprev.c @@ -36,7 +36,7 @@ int mi_rprev(MI_INFO *info, byte *buf, int inx) if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_NEXT_FOUND) flag=0; /* Read last */ - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(my_errno); changed=_mi_test_if_changed(info); if (share->concurrent_insert) @@ -50,7 +50,7 @@ int mi_rprev(MI_INFO *info, byte *buf, int inx) share->state.key_root[inx]); else error=_mi_search(info,share->keyinfo+inx,info->lastkey, - info->lastkey_length, flag, share->state.key_root[inx]); + USE_WHOLE_KEY, flag, share->state.key_root[inx]); if (!error) { diff --git a/myisam/mi_rrnd.c b/myisam/mi_rrnd.c index 11fa2af59bd..f8009441cff 100644 --- a/myisam/mi_rrnd.c +++ b/myisam/mi_rrnd.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_rsame.c b/myisam/mi_rsame.c index a4092b53c0b..56c8d1226ca 100644 --- a/myisam/mi_rsame.c +++ b/myisam/mi_rsame.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -41,7 +41,7 @@ int mi_rsame(MI_INFO *info, byte *record, int inx) info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); /* Read row from data file */ - if (_mi_readinfo(info,F_RDLCK,1)) + if (fast_mi_readinfo(info)) DBUG_RETURN(my_errno); if (inx >= 0) @@ -51,7 +51,8 @@ int mi_rsame(MI_INFO *info, byte *record, int inx) info->lastpos); if (info->s->concurrent_insert) rw_rdlock(&info->s->key_root_lock[inx]); - VOID(_mi_search(info,info->s->keyinfo+inx,info->lastkey,0,SEARCH_SAME, + VOID(_mi_search(info,info->s->keyinfo+inx,info->lastkey, USE_WHOLE_KEY, + SEARCH_SAME, info->s->state.key_root[inx])); if (info->s->concurrent_insert) rw_unlock(&info->s->key_root_lock[inx]); diff --git a/myisam/mi_rsamepos.c b/myisam/mi_rsamepos.c index 28a5b6783b2..a1d96fb7104 100644 --- a/myisam/mi_rsamepos.c +++ b/myisam/mi_rsamepos.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_scan.c b/myisam/mi_scan.c index c06f092ab17..90bc3430ba7 100644 --- a/myisam/mi_scan.c +++ b/myisam/mi_scan.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ diff --git a/myisam/mi_search.c b/myisam/mi_search.c index 938062d977d..41d53e76241 100644 --- a/myisam/mi_search.c +++ b/myisam/mi_search.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -19,29 +19,27 @@ #include "fulltext.h" #include "m_ctype.h" -#define CMP(a,b) (a<b ? -1 : a == b ? 0 : 1) - static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, - uchar *key, uchar *keypos, - uint *return_key_length); + uchar *key, uchar *keypos, + uint *return_key_length); - /* Check index */ + /* Check index */ int _mi_check_index(MI_INFO *info, int inx) { - if (inx == -1) /* Use last index */ + if (inx == -1) /* Use last index */ inx=info->lastinx; if (inx < 0 || ! (((ulonglong) 1 << inx) & info->s->state.key_map)) { my_errno=HA_ERR_WRONG_INDEX; return -1; } - if (info->lastinx != inx) /* Index changed */ + if (info->lastinx != inx) /* Index changed */ { info->lastinx = inx; info->page_changed=1; info->update= ((info->update & (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED)) | - HA_STATE_NEXT_FOUND | HA_STATE_PREV_FOUND); + HA_STATE_NEXT_FOUND | HA_STATE_PREV_FOUND); } if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) return(-1); @@ -49,15 +47,15 @@ int _mi_check_index(MI_INFO *info, int inx) } /* mi_check_index */ - /* - ** Search after row by a key - ** Position to row is stored in info->lastpos - ** Return: -1 if not found - ** 1 if one should continue search on higher level - */ + /* + ** Search after row by a key + ** Position to row is stored in info->lastpos + ** Return: -1 if not found + ** 1 if one should continue search on higher level + */ int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, - uchar *key, uint key_len, uint nextflag, register my_off_t pos) + uchar *key, uint key_len, uint nextflag, register my_off_t pos) { my_bool last_key; int error,flag; @@ -66,25 +64,25 @@ int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, uchar lastkey[MI_MAX_KEY_BUFF],*buff; DBUG_ENTER("_mi_search"); DBUG_PRINT("enter",("pos: %ld nextflag: %d lastpos: %ld", - pos,nextflag,info->lastpos)); + pos,nextflag,info->lastpos)); DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_len);); if (pos == HA_OFFSET_ERROR) { - my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ info->lastpos= HA_OFFSET_ERROR; if (!(nextflag & (SEARCH_SMALLER | SEARCH_BIGGER | SEARCH_LAST))) - DBUG_RETURN(-1); /* Not found ; return error */ - DBUG_RETURN(1); /* Search at upper levels */ + DBUG_RETURN(-1); /* Not found ; return error */ + DBUG_RETURN(1); /* Search at upper levels */ } if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,info->buff, - test(!(nextflag & SEARCH_SAVE_BUFF))))) + test(!(nextflag & SEARCH_SAVE_BUFF))))) goto err; DBUG_DUMP("page",(byte*) buff,mi_getint(buff)); flag=(*keyinfo->bin_search)(info,keyinfo,buff,key,key_len,nextflag, - &keypos,lastkey, &last_key); + &keypos,lastkey, &last_key); if (flag == MI_FOUND_WRONG_KEY) DBUG_RETURN(-1); nod_flag=mi_test_if_nod(buff); @@ -93,36 +91,36 @@ int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, if (flag) { if ((error=_mi_search(info,keyinfo,key,key_len,nextflag, - _mi_kpos(nod_flag,keypos))) <= 0) + _mi_kpos(nod_flag,keypos))) <= 0) DBUG_RETURN(error); if (flag >0) { if (nextflag & (SEARCH_SMALLER | SEARCH_LAST) && - keypos == buff+2+nod_flag) - DBUG_RETURN(1); /* Bigger than key */ + keypos == buff+2+nod_flag) + DBUG_RETURN(1); /* Bigger than key */ } else if (nextflag & SEARCH_BIGGER && keypos >= maxpos) - DBUG_RETURN(1); /* Smaller than key */ + DBUG_RETURN(1); /* Smaller than key */ } else { if ((nextflag & SEARCH_FIND) && nod_flag && ((keyinfo->flag & (HA_NOSAME | HA_NULL_PART)) != HA_NOSAME || - key_len)) + key_len != USE_WHOLE_KEY)) { if ((error=_mi_search(info,keyinfo,key,key_len,SEARCH_FIND, - _mi_kpos(nod_flag,keypos))) >= 0 || - my_errno != HA_ERR_KEY_NOT_FOUND) - DBUG_RETURN(error); - info->last_keypage= HA_OFFSET_ERROR; /* Buffer not in memory */ + _mi_kpos(nod_flag,keypos))) >= 0 || + my_errno != HA_ERR_KEY_NOT_FOUND) + DBUG_RETURN(error); + info->last_keypage= HA_OFFSET_ERROR; /* Buffer not in mem */ } } if (pos != info->last_keypage) { uchar *old_buff=buff; if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,info->buff, - test(!(nextflag & SEARCH_SAVE_BUFF))))) + test(!(nextflag & SEARCH_SAVE_BUFF))))) goto err; keypos=buff+(keypos-old_buff); maxpos=buff+(maxpos-old_buff); @@ -132,13 +130,13 @@ int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, { uint not_used; if (_mi_get_prev_key(info,keyinfo, buff, info->lastkey, keypos, - &info->lastkey_length)) + &info->lastkey_length)) goto err; if ((nextflag & SEARCH_LAST) && - _mi_key_cmp(keyinfo->seg, info->lastkey, key, key_len, SEARCH_FIND, - ¬_used)) + _mi_key_cmp(keyinfo->seg, info->lastkey, key, key_len, SEARCH_FIND, + ¬_used)) { - my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ goto err; } } @@ -157,7 +155,7 @@ int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, info->int_keytree_version=keyinfo->version; info->last_search_keypage=info->last_keypage; info->page_changed=0; - info->buff_used= (info->buff != buff); /* If we have to reread buff */ + info->buff_used= (info->buff != buff); /* If we have to reread buff */ DBUG_PRINT("exit",("found key at %lu",(ulong) info->lastpos)); DBUG_RETURN(0); @@ -169,14 +167,14 @@ err: } /* _mi_search */ - /* Search after key in page-block */ - /* If packed key puts smaller or identical key in buff */ - /* ret_pos point to where find or bigger key starts */ - /* ARGSUSED */ + /* Search after key in page-block */ + /* If packed key puts smaller or identical key in buff */ + /* ret_pos point to where find or bigger key starts */ + /* ARGSUSED */ int _mi_bin_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, - uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, - uchar *buff __attribute__((unused)), my_bool *last_key) + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff __attribute__((unused)), my_bool *last_key) { reg4 int start,mid,end,save_end; int flag; @@ -194,17 +192,17 @@ int _mi_bin_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, { mid= (start+end)/2; if ((flag=_mi_key_cmp(keyinfo->seg,page+(uint) mid*totlength,key,key_len, - comp_flag,¬_used)) - >= 0) + comp_flag,¬_used)) + >= 0) end=mid; else start=mid+1; } if (mid != start) flag=_mi_key_cmp(keyinfo->seg,page+(uint) start*totlength,key,key_len, - comp_flag,¬_used); + comp_flag,¬_used); if (flag < 0) - start++; /* point at next, bigger key */ + start++; /* point at next, bigger key */ *ret_pos=page+(uint) start*totlength; *last_key= end == save_end; DBUG_PRINT("exit",("flag: %d keypos: %d",flag,start)); @@ -212,13 +210,13 @@ int _mi_bin_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, } /* _mi_bin_search */ - /* Used instead of _mi_bin_search() when key is packed */ - /* Puts smaller or identical key in buff */ - /* Key is searched sequentially */ + /* Used instead of _mi_bin_search() when key is packed */ + /* Puts smaller or identical key in buff */ + /* Key is searched sequentially */ int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, - uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, - uchar *buff, my_bool *last_key) + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff, my_bool *last_key) { int flag; uint nod_flag,length,not_used; @@ -230,7 +228,7 @@ int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, nod_flag=mi_test_if_nod(page); page+=2+nod_flag; *ret_pos=page; - t_buff[0]=0; /* Avoid bugs */ + t_buff[0]=0; /* Avoid bugs */ while (page < end) { length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,t_buff); @@ -238,11 +236,11 @@ int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, { my_errno=HA_ERR_CRASHED; DBUG_PRINT("error",("Found wrong key: length: %d page: %lx end: %lx", - length,page,end)); + length,page,end)); DBUG_RETURN(MI_FOUND_WRONG_KEY); } if ((flag=_mi_key_cmp(keyinfo->seg,t_buff,key,key_len,comp_flag, - ¬_used)) >= 0) + ¬_used)) >= 0) break; #ifdef EXTRA_DEBUG DBUG_PRINT("loop",("page: %lx key: '%s' flag: %d",page,t_buff,flag)); @@ -251,14 +249,226 @@ int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, *ret_pos=page; } if (flag == 0) - memcpy(buff,t_buff,length); /* Result is first key */ + memcpy(buff,t_buff,length); /* Result is first key */ *last_key= page == end; DBUG_PRINT("exit",("flag: %d ret_pos: %lx",flag,*ret_pos)); DBUG_RETURN(flag); } /* _mi_seq_search */ - /* Get pos to a key_block */ +int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint nextflag, uchar **ret_pos, + uchar *buff, my_bool *last_key) +{ + /* + my_flag is raw comparison result to be changed according to + SEARCH_NO_FIND,SEARCH_LAST and HA_REVERSE_SORT flags. + flag is the value returned by _mi_key_cmp and as treated as final + */ + int flag=0, my_flag=-1; + uint nod_flag, length, len, matched, cmplen, kseg_len; + uint prefix_len,suffix_len; + int key_len_skip, seg_len_pack, key_len_left; + uchar *end, *kseg, *vseg; + uchar *sort_order=keyinfo->seg->charset->sort_order; + uchar tt_buff[MI_MAX_KEY_BUFF+2], *t_buff=tt_buff+2; + uchar *saved_from, *saved_to, *saved_vseg; + uint saved_length=0, saved_prefix_len=0; + DBUG_ENTER("_mi_prefix_search"); + + LINT_INIT(length); + LINT_INIT(prefix_len); + LINT_INIT(seg_len_pack); + LINT_INIT(saved_from); + LINT_INIT(saved_to); + LINT_INIT(saved_vseg); + + t_buff[0]=0; /* Avoid bugs */ + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + *ret_pos=page; + kseg=key; + { + uint lenght_pack; + get_key_pack_length(kseg_len,lenght_pack,kseg); + key_len_skip=lenght_pack+kseg_len; + key_len_left=(int) key_len- (int) key_len_skip; + cmplen=(key_len_left>=0) ? kseg_len : key_len-lenght_pack; + DBUG_PRINT("info",("key: '%.*s'",kseg_len,kseg)); + } + +/* + Keys are compressed the following way: + + If the max length of first key segment <= 127 characters the prefix is + 1 byte else it's 2 byte + + prefix The high bit is set if this is a prefix for the prev key + length Packed length if the previous was a prefix byte + [length] Length character of data + next-key-seg Next key segments +*/ + + matched=0; /* how many char's from prefix were alredy matched */ + len=0; /* length of previous key unpacked */ + + while (page < end) + { + uint packed= *page & 128; + + vseg=page; + if (keyinfo->seg->length >= 127) + { + suffix_len=mi_uint2korr(vseg) & 32767; + vseg+=2; + } + else + suffix_len= *vseg++ & 127; + + if (packed) + { + if (suffix_len == 0) /* Same key */ + prefix_len=len; + else + { + prefix_len=suffix_len; + get_key_length(suffix_len,vseg); + } + } + else + prefix_len=0; + + len=prefix_len+suffix_len; + seg_len_pack=get_pack_length(len); + t_buff=tt_buff+3-seg_len_pack; + store_key_length(t_buff,len); + + if (prefix_len > saved_prefix_len) + memcpy(t_buff+seg_len_pack+saved_prefix_len,saved_vseg, + prefix_len-saved_prefix_len); + saved_vseg=vseg; + saved_prefix_len=prefix_len; + + DBUG_PRINT("loop",("page: '%.*s%.*s'",prefix_len,t_buff+seg_len_pack,suffix_len,vseg)); + { + uchar *from=vseg+suffix_len; + MI_KEYSEG *keyseg; + uint l; + + for (keyseg=keyinfo->seg+1 ; keyseg->type ; keyseg++ ) + { + + if (keyseg->flag & HA_NULL_PART) + { + if (!(*from++)) + continue; + } + if (keyseg->flag & (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) + { + get_key_length(l,from); + } + else + l=keyseg->length; + + from+=l; + } + from+=keyseg->length; + page=from+nod_flag; + length=from-vseg; + } + + if (page > end) + { + my_errno=HA_ERR_CRASHED; + DBUG_PRINT("error",("Found wrong key: length: %d page: %lx end: %lx", + length,page,end)); + DBUG_RETURN(MI_FOUND_WRONG_KEY); + } + + if (matched >= prefix_len) + { + /* We have to compare. But we can still skip part of the key */ + uint left; + uchar *k=kseg+prefix_len; + + left=(len>cmplen) ? cmplen-prefix_len : suffix_len; + + matched=prefix_len+left; + + for(my_flag=0;left;left--) + if ((my_flag= (int) sort_order[*vseg++] - (int) sort_order[*k++])) + break; + + if (my_flag>0) /* mismatch */ + break; + else if (my_flag==0) /* match */ + { /* + ** len cmplen seg_left_len more_segs + ** < matched=len; continue search + ** > = prefix ? found : (matched=len; continue search) + ** > < - ok, found + ** = < - ok, found + ** = = - ok, found + ** = = + next seg + */ + if (len < cmplen) + { + my_flag= -1; + } + else if (len > cmplen) + { + if ((my_flag= (!(nextflag & SEARCH_PREFIX) || key_len_left>0))) + break; + goto fix_flag; + } + else if (key_len_left>0) + { + uint not_used; + if ((flag = _mi_key_cmp(keyinfo->seg+1,vseg, + k,key_len_left,nextflag,¬_used)) >= 0) + break; + } + else + { + /* at this line flag==-1 if the following lines were already + visited and 0 otherwise, i.e. flag <=0 here always !!! */ + fix_flag: + if (nextflag & (SEARCH_NO_FIND | SEARCH_LAST)) + flag=(nextflag & (SEARCH_BIGGER | SEARCH_LAST)) ? -1 : 1; + if (flag>=0) break; + } + } + matched-=left; + } + /* else (matched < prefix_len) ---> do nothing. */ + + memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len); + saved_to=buff+saved_length; + saved_from=saved_vseg; + saved_length=length; + *ret_pos=page; + } + if (my_flag) + flag=(keyinfo->seg->flag & HA_REVERSE_SORT) ? -my_flag : my_flag; + if (flag == 0) + { + memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len); + saved_to=buff+saved_length; + saved_from=saved_vseg; + saved_length=length; + } + if (saved_length) + memcpy(saved_to,saved_from,saved_length); + + *last_key= page == end; + + DBUG_PRINT("exit",("flag: %d ret_pos: %lx",flag,*ret_pos)); + DBUG_RETURN(flag); +} /* _mi_prefix_search */ + + + /* Get pos to a key_block */ my_off_t _mi_kpos(uint nod_flag, uchar *after_key) { @@ -266,11 +476,11 @@ my_off_t _mi_kpos(uint nod_flag, uchar *after_key) switch (nod_flag) { #if SIZEOF_OFF_T > 4 case 7: - return mi_uint7korr(after_key)*MI_KEY_BLOCK_LENGTH; + return mi_uint7korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; case 6: - return mi_uint6korr(after_key)*MI_KEY_BLOCK_LENGTH; + return mi_uint6korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; case 5: - return mi_uint5korr(after_key)*MI_KEY_BLOCK_LENGTH; + return mi_uint5korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; #else case 7: after_key++; @@ -280,25 +490,25 @@ my_off_t _mi_kpos(uint nod_flag, uchar *after_key) after_key++; #endif case 4: - return ((my_off_t) mi_uint4korr(after_key))*MI_KEY_BLOCK_LENGTH; + return ((my_off_t) mi_uint4korr(after_key))*MI_MIN_KEY_BLOCK_LENGTH; case 3: - return ((my_off_t) mi_uint3korr(after_key))*MI_KEY_BLOCK_LENGTH; + return ((my_off_t) mi_uint3korr(after_key))*MI_MIN_KEY_BLOCK_LENGTH; case 2: - return (my_off_t) (mi_uint2korr(after_key)*MI_KEY_BLOCK_LENGTH); + return (my_off_t) (mi_uint2korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH); case 1: - return (uint) (*after_key)*MI_KEY_BLOCK_LENGTH; - case 0: /* At leaf page */ - default: /* Impossible */ + return (uint) (*after_key)*MI_MIN_KEY_BLOCK_LENGTH; + case 0: /* At leaf page */ + default: /* Impossible */ return(HA_OFFSET_ERROR); } } /* _kpos */ - /* Save pos to a key_block */ + /* Save pos to a key_block */ void _mi_kpointer(register MI_INFO *info, register uchar *buff, my_off_t pos) { - pos/=MI_KEY_BLOCK_LENGTH; + pos/=MI_MIN_KEY_BLOCK_LENGTH; switch (info->s->base.key_reflength) { #if SIZEOF_OFF_T > 4 case 7: mi_int7store(buff,pos); break; @@ -316,12 +526,12 @@ void _mi_kpointer(register MI_INFO *info, register uchar *buff, my_off_t pos) case 3: mi_int3store(buff,pos); break; case 2: mi_int2store(buff,(uint) pos); break; case 1: buff[0]= (uchar) pos; break; - default: abort(); /* impossible */ + default: abort(); /* impossible */ } } /* _mi_kpointer */ - /* Calc pos to a data-record from a key */ + /* Calc pos to a data-record from a key */ my_off_t _mi_dpos(MI_INFO *info, uint nod_flag, uchar *after_key) @@ -336,19 +546,19 @@ my_off_t _mi_dpos(MI_INFO *info, uint nod_flag, uchar *after_key) case 5: pos= (my_off_t) mi_uint5korr(after_key); break; #else case 8: pos= (my_off_t) mi_uint4korr(after_key+4); break; - case 7: pos= (my_off_t) mi_uint4korr(after_key+3); break; - case 6: pos= (my_off_t) mi_uint4korr(after_key+2); break; - case 5: pos= (my_off_t) mi_uint4korr(after_key+1); break; + case 7: pos= (my_off_t) mi_uint4korr(after_key+3); break; + case 6: pos= (my_off_t) mi_uint4korr(after_key+2); break; + case 5: pos= (my_off_t) mi_uint4korr(after_key+1); break; #endif case 4: pos= (my_off_t) mi_uint4korr(after_key); break; case 3: pos= (my_off_t) mi_uint3korr(after_key); break; case 2: pos= (my_off_t) mi_uint2korr(after_key); break; default: - pos=0L; /* Shut compiler up */ + pos=0L; /* Shut compiler up */ } return (info->s->options & - (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : - pos*info->s->base.pack_reclength; + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*info->s->base.pack_reclength; } @@ -362,22 +572,22 @@ my_off_t _mi_rec_pos(MYISAM_SHARE *s, uchar *ptr) case 8: pos= (my_off_t) mi_uint8korr(ptr); if (pos == HA_OFFSET_ERROR) - return HA_OFFSET_ERROR; /* end of list */ + return HA_OFFSET_ERROR; /* end of list */ break; case 7: pos= (my_off_t) mi_uint7korr(ptr); if (pos == (((my_off_t) 1) << 56) -1) - return HA_OFFSET_ERROR; /* end of list */ + return HA_OFFSET_ERROR; /* end of list */ break; case 6: pos= (my_off_t) mi_uint6korr(ptr); if (pos == (((my_off_t) 1) << 48) -1) - return HA_OFFSET_ERROR; /* end of list */ + return HA_OFFSET_ERROR; /* end of list */ break; case 5: pos= (my_off_t) mi_uint5korr(ptr); if (pos == (((my_off_t) 1) << 40) -1) - return HA_OFFSET_ERROR; /* end of list */ + return HA_OFFSET_ERROR; /* end of list */ break; #else case 8: @@ -402,20 +612,20 @@ my_off_t _mi_rec_pos(MYISAM_SHARE *s, uchar *ptr) if (pos == (my_off_t) (1 << 16) -1) return HA_OFFSET_ERROR; break; - default: abort(); /* Impossible */ + default: abort(); /* Impossible */ } return ((s->options & - (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : - pos*s->base.pack_reclength); + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*s->base.pack_reclength); } - /* save position to record */ + /* save position to record */ void _mi_dpointer(MI_INFO *info, uchar *buff, my_off_t pos) { if (!(info->s->options & - (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && pos != HA_OFFSET_ERROR) pos/=info->s->base.pack_reclength; @@ -438,27 +648,28 @@ void _mi_dpointer(MI_INFO *info, uchar *buff, my_off_t pos) case 4: mi_int4store(buff,pos); break; case 3: mi_int3store(buff,pos); break; case 2: mi_int2store(buff,(uint) pos); break; - default: abort(); /* Impossible */ + default: abort(); /* Impossible */ } } /* _mi_dpointer */ int _mi_compare_text(CHARSET_INFO *charset_info, uchar *a, uint a_length, - uchar *b, uint b_length, my_bool part_key) + uchar *b, uint b_length, my_bool part_key) { - uint length= min(a_length,b_length); - uchar *end= a+ length; int flag; #ifdef USE_STRCOLL if (use_strcoll(charset_info)) { - if ((flag = my_strnncoll(charset_info, a, a_length, b, b_length))) - return flag; + if (part_key && b_length < a_length) + a_length=b_length; + return my_strnncoll(charset_info, a, a_length, b, b_length); } else #endif { + uint length= min(a_length,b_length); + uchar *end= a+ length; uchar *sort_order=charset_info->sort_order; while (a < end) if ((flag= (int) sort_order[*a++] - (int) sort_order[*b++])) @@ -471,7 +682,7 @@ int _mi_compare_text(CHARSET_INFO *charset_info, uchar *a, uint a_length, static int compare_bin(uchar *a, uint a_length, uchar *b, uint b_length, - my_bool part_key) + my_bool part_key) { uint length= min(a_length,b_length); uchar *end= a+ length; @@ -486,19 +697,35 @@ static int compare_bin(uchar *a, uint a_length, uchar *b, uint b_length, } - /* - ** Compare two keys with is bigger - ** Returns <0, 0, >0 acording to with is bigger - ** Key_length specifies length of key to use. Number-keys can't - ** be splited - ** If flag <> SEARCH_FIND compare also position - */ +/* + Compare two keys + + SYNOPSIS + _mi_key_cmp() + keyseg Key segments of key to compare + a First key to compare, in format from _mi_pack_key() + This is normally key specified by user + b Second key to compare. This is always from a row + key_length Length of key to compare. This can be shorter than + a to just compare sub keys + next_flag How keys should be compared + If bit SEARCH_FIND is not set the keys includes the row + position and this should also be compared + + NOTES + Number-keys can't be splited + + RETURN VALUES + <0 If a < b + 0 If a == b + >0 If a > b +*/ #define FCMP(A,B) ((int) (A) - (int) (B)) int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, - register uchar *b, uint key_length, uint nextflag, - uint *diff_pos) + register uchar *b, uint key_length, uint nextflag, + uint *diff_pos) { int flag; int16 s_1,s_2; @@ -508,129 +735,143 @@ int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, double d_1,d_2; uint next_key_length; - if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) - key_length=USE_WHOLE_KEY; *diff_pos=0; - for ( ; (int) key_length >0 ; key_length=next_key_length, keyseg++) { uchar *end; + uint piks=! (keyseg->flag & HA_NO_SORT); (*diff_pos)++; /* Handle NULL part */ if (keyseg->null_bit) { key_length--; - if (*a != *b) + if (*a != *b && piks) { - flag = (int) *a - (int) *b; - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + flag = (int) *a - (int) *b; + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); } b++; - if (!*a++) /* If key was NULL */ + if (!*a++) /* If key was NULL */ { - if (nextflag == (SEARCH_FIND | SEARCH_UPDATE)) - nextflag=SEARCH_SAME; /* Allow duplicate keys */ - next_key_length=key_length; - continue; /* To next key part */ + if (nextflag == (SEARCH_FIND | SEARCH_UPDATE)) + nextflag=SEARCH_SAME; /* Allow duplicate keys */ + else if (nextflag & SEARCH_NULL_ARE_NOT_EQUAL) + { + /* + This is only used from mi_check() to calculate cardinality. + It can't be used when searching for a key as this would cause + compare of (a,b) and (b,a) to return the same value. + */ + return -1; + } + next_key_length=key_length; + continue; /* To next key part */ } } end= a+ min(keyseg->length,key_length); next_key_length=key_length-keyseg->length; switch ((enum ha_base_keytype) keyseg->type) { - case HA_KEYTYPE_TEXT: /* Ascii; Key is converted */ + case HA_KEYTYPE_TEXT: /* Ascii; Key is converted */ if (keyseg->flag & HA_SPACE_PACK) { - int a_length,b_length,pack_length; - get_key_length(a_length,a); - get_key_pack_length(b_length,pack_length,b); - next_key_length=key_length-b_length-pack_length; - - if ((flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a+=a_length; - b+=b_length; - break; + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if (piks && + (flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; } else { - uint length=(uint) (end-a); - if ((flag=_mi_compare_text(keyseg->charset,a,length,b,length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a=end; - b+=length; + uint length=(uint) (end-a), a_length=length, b_length=length; + if (!(nextflag & SEARCH_PREFIX)) + { + while (a_length && a[a_length-1] == ' ') + a_length--; + while (b_length && b[b_length-1] == ' ') + b_length--; + } + if (piks && + (flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a=end; + b+=length; } break; case HA_KEYTYPE_BINARY: if (keyseg->flag & HA_SPACE_PACK) { - int a_length,b_length,pack_length; - get_key_length(a_length,a); - get_key_pack_length(b_length,pack_length,b); - next_key_length=key_length-b_length-pack_length; - - if ((flag=compare_bin(a,a_length,b,b_length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a+=a_length; - b+=b_length; - break; + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if (piks && + (flag=compare_bin(a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; } else { - uint length=keyseg->length; - if ((flag=compare_bin(a,length,b,length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a+=length; - b+=length; + uint length=keyseg->length; + if (piks && + (flag=compare_bin(a,length,b,length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=length; + b+=length; } break; case HA_KEYTYPE_VARTEXT: { - int a_length,b_length,pack_length; - get_key_length(a_length,a); - get_key_pack_length(b_length,pack_length,b); - next_key_length=key_length-b_length-pack_length; - - if ((flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a+=a_length; - b+=b_length; - break; + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if (piks && + (flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; } break; case HA_KEYTYPE_VARBINARY: { - int a_length,b_length,pack_length; - get_key_length(a_length,a); - get_key_pack_length(b_length,pack_length,b); - next_key_length=key_length-b_length-pack_length; - - if ((flag=compare_bin(a,a_length,b,b_length, - (my_bool) ((nextflag & SEARCH_PREFIX) && - next_key_length <= 0)))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a+=a_length; - b+=b_length; - break; + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if (piks && + (flag=compare_bin(a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; } break; case HA_KEYTYPE_INT8: { int i_1= (int) *((signed char*) a); int i_2= (int) *((signed char*) b); - if ((flag = CMP(i_1,i_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(i_1,i_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b++; break; @@ -638,125 +879,132 @@ int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, case HA_KEYTYPE_SHORT_INT: s_1= mi_sint2korr(a); s_2= mi_sint2korr(b); - if ((flag = CMP(s_1,s_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(s_1,s_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 2; /* sizeof(short int); */ break; case HA_KEYTYPE_USHORT_INT: { - uint16 us_1,us_2; - us_1= mi_sint2korr(a); - us_2= mi_sint2korr(b); - if ((flag = CMP(us_1,us_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); - a= end; - b+=2; /* sizeof(short int); */ - break; + uint16 us_1,us_2; + us_1= mi_sint2korr(a); + us_2= mi_sint2korr(b); + if (piks && (flag = CMP_NUM(us_1,us_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+=2; /* sizeof(short int); */ + break; } case HA_KEYTYPE_LONG_INT: l_1= mi_sint4korr(a); l_2= mi_sint4korr(b); - if ((flag = CMP(l_1,l_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 4; /* sizeof(long int); */ break; case HA_KEYTYPE_ULONG_INT: u_1= mi_sint4korr(a); u_2= mi_sint4korr(b); - if ((flag = CMP(u_1,u_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(u_1,u_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 4; /* sizeof(long int); */ break; case HA_KEYTYPE_INT24: l_1=mi_sint3korr(a); l_2=mi_sint3korr(b); - if ((flag = CMP(l_1,l_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 3; break; case HA_KEYTYPE_UINT24: l_1=mi_uint3korr(a); l_2=mi_uint3korr(b); - if ((flag = CMP(l_1,l_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 3; break; case HA_KEYTYPE_FLOAT: mi_float4get(f_1,a); mi_float4get(f_2,b); - if ((flag = CMP(f_1,f_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(f_1,f_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 4; /* sizeof(float); */ break; case HA_KEYTYPE_DOUBLE: mi_float8get(d_1,a); mi_float8get(d_2,b); - if ((flag = CMP(d_1,d_2))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(d_1,d_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 8; /* sizeof(double); */ break; - case HA_KEYTYPE_NUM: /* Numeric key */ + case HA_KEYTYPE_NUM: /* Numeric key */ { int swap_flag= 0; int alength,blength; - + if (keyseg->flag & HA_REVERSE_SORT) { - swap(uchar*,a,b); - swap_flag=1; /* Remember swap of a & b */ + swap(uchar*,a,b); + swap_flag=1; /* Remember swap of a & b */ end= a+ (int) (end-b); } if (keyseg->flag & HA_SPACE_PACK) { - alength= *a++; blength= *b++; - end=a+alength; - next_key_length=key_length-blength-1; + alength= *a++; blength= *b++; + end=a+alength; + next_key_length=key_length-blength-1; } else { - alength= (int) (end-a); - blength=keyseg->length; - /* remove pre space from keys */ - for ( ; alength && *a == ' ' ; a++, alength--) ; - for ( ; blength && *b == ' ' ; b++, blength--) ; - } - - if (*a == '-') - { - if (*b != '-') - return -1; - a++; b++; - swap(uchar*,a,b); - swap(int,alength,blength); - swap_flag=1-swap_flag; - alength--; blength--; - end=a+alength; + alength= (int) (end-a); + blength=keyseg->length; + /* remove pre space from keys */ + for ( ; alength && *a == ' ' ; a++, alength--) ; + for ( ; blength && *b == ' ' ; b++, blength--) ; } - else if (*b == '-') - return 1; - while (alength && (*a == '+' || *a == '0')) + if (piks) { - a++; alength--; + if (*a == '-') + { + if (*b != '-') + return -1; + a++; b++; + swap(uchar*,a,b); + swap(int,alength,blength); + swap_flag=1-swap_flag; + alength--; blength--; + end=a+alength; + } + else if (*b == '-') + return 1; + while (alength && (*a == '+' || *a == '0')) + { + a++; alength--; + } + while (blength && (*b == '+' || *b == '0')) + { + b++; blength--; + } + if (alength != blength) + return (alength < blength) ? -1 : 1; + while (a < end) + if (*a++ != *b++) + return ((int) a[-1] - (int) b[-1]); } - while (blength && (*b == '+' || *b == '0')) + else { - b++; blength--; + b+=(end-a); + a=end; } - if (alength != blength) - return (alength < blength) ? -1 : 1; - while (a < end) - if (*a++ != *b++) - return ((int) a[-1] - (int) b[-1]); - - if (swap_flag) /* Restore pointers */ - swap(uchar*,a,b); + + if (swap_flag) /* Restore pointers */ + swap(uchar*,a,b); break; } #ifdef HAVE_LONG_LONG @@ -765,8 +1013,8 @@ int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, longlong ll_a,ll_b; ll_a= mi_sint8korr(a); ll_b= mi_sint8korr(b); - if ((flag = CMP(ll_a,ll_b))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(ll_a,ll_b))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 8; break; @@ -776,15 +1024,15 @@ int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, ulonglong ll_a,ll_b; ll_a= mi_uint8korr(a); ll_b= mi_uint8korr(b); - if ((flag = CMP(ll_a,ll_b))) - return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + if (piks && (flag = CMP_NUM(ll_a,ll_b))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); a= end; b+= 8; break; } #endif - case HA_KEYTYPE_END: /* Ready */ - goto end; /* diff_pos is incremented */ + case HA_KEYTYPE_END: /* Ready */ + goto end; /* diff_pos is incremented */ } } (*diff_pos)++; @@ -799,33 +1047,33 @@ end: { if (*a++ != *b++) { - flag= FCMP(a[-1],b[-1]); - break; + flag= FCMP(a[-1],b[-1]); + break; } } if (nextflag & SEARCH_SAME) - return (flag); /* read same */ + return (flag); /* read same */ if (nextflag & SEARCH_BIGGER) - return (flag <= 0 ? -1 : 1); /* read next */ - return (flag < 0 ? -1 : 1); /* read previous */ + return (flag <= 0 ? -1 : 1); /* read next */ + return (flag < 0 ? -1 : 1); /* read previous */ } return 0; } /* _mi_key_cmp */ - /* Get key from key-block */ - /* page points at previous key; its advanced to point at next key */ - /* key should contain previous key */ - /* Returns length of found key + pointers */ - /* nod_flag is a flag if we are on nod */ + /* Get key from key-block */ + /* page points at previous key; its advanced to point at next key */ + /* key should contain previous key */ + /* Returns length of found key + pointers */ + /* nod_flag is a flag if we are on nod */ - /* same as _mi_get_key but used with fixed length keys */ + /* same as _mi_get_key but used with fixed length keys */ uint _mi_get_static_key(register MI_KEYDEF *keyinfo, uint nod_flag, - register uchar **page, register uchar *key) + register uchar **page, register uchar *key) { memcpy((byte*) key,(byte*) *page, - (size_t) (keyinfo->keylength+nod_flag)); + (size_t) (keyinfo->keylength+nod_flag)); *page+=keyinfo->keylength+nod_flag; return(keyinfo->keylength); } /* _mi_get_static_key */ @@ -834,7 +1082,7 @@ uint _mi_get_static_key(register MI_KEYDEF *keyinfo, uint nod_flag, /* Key with is packed against previous key or key with a NULL column */ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, - register uchar **page_pos, register uchar *key) + register uchar **page_pos, register uchar *key) { reg1 MI_KEYSEG *keyseg; uchar *start_key,*page=*page_pos; @@ -850,11 +1098,11 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, uint packed= *page & 128,tot_length,rest_length; if (keyseg->length >= 127) { - length=mi_uint2korr(page) & 32767; - page+=2; + length=mi_uint2korr(page) & 32767; + page+=2; } else - length= *page++ & 127; + length= *page++ & 127; if (packed) { @@ -915,23 +1163,23 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, } else { - if (keyseg->flag & HA_NULL_PART) - { - if (!length--) /* Null part */ - { - *key++=0; - continue; - } - *key++=1; /* Not null */ - } + if (keyseg->flag & HA_NULL_PART) + { + if (!length--) /* Null part */ + { + *key++=0; + continue; + } + *key++=1; /* Not null */ + } } if (length > (uint) keyseg->length) { - DBUG_PRINT("error",("Found too long packed key: %d of %d at %lx", - length, keyseg->length, *page_pos)); - DBUG_DUMP("key",(char*) *page_pos,16); - my_errno=HA_ERR_CRASHED; - return 0; /* Error */ + DBUG_PRINT("error",("Found too long packed key: %d of %d at %lx", + length, keyseg->length, *page_pos)); + DBUG_DUMP("key",(char*) *page_pos,16); + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ } store_key_length_inc(key,length); } @@ -939,18 +1187,18 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, { if (keyseg->flag & HA_NULL_PART) { - if (!(*key++ = *page++)) - continue; + if (!(*key++ = *page++)) + continue; } if (keyseg->flag & - (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) + (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) { - uchar *tmp=page; - get_key_length(length,tmp); - length+=(uint) (tmp-page); + uchar *tmp=page; + get_key_length(length,tmp); + length+=(uint) (tmp-page); } else - length=keyseg->length; + length=keyseg->length; } memcpy((byte*) key,(byte*) page,(size_t) length); key+=length; @@ -967,7 +1215,7 @@ uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, /* key that is packed relatively to previous */ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, - register uchar **page_pos, register uchar *key) + register uchar **page_pos, register uchar *key) { reg1 MI_KEYSEG *keyseg; uchar *start_key,*page=*page_pos,*page_end,*from,*from_end; @@ -982,16 +1230,16 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, if (length > keyinfo->maxlength) { DBUG_PRINT("error",("Found too long binary packed key: %d of %d at %lx", - length, keyinfo->maxlength, *page_pos)); + length, keyinfo->maxlength, *page_pos)); DBUG_DUMP("key",(char*) *page_pos,16); my_errno=HA_ERR_CRASHED; - return 0; /* Wrong key */ + return 0; /* Wrong key */ } from=key; from_end=key+length; } else { - from=page; from_end=page_end; /* Not packed key */ + from=page; from_end=page_end; /* Not packed key */ } /* @@ -1005,7 +1253,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, { if (from == from_end) { from=page; from_end=page_end; } if (!(*key++ = *from++)) - continue; /* Null part */ + continue; /* Null part */ } if (keyseg->flag & (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) { @@ -1013,10 +1261,10 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, if (from == from_end) { from=page; from_end=page_end; } if ((length= (*key++ = *from++)) == 255) { - if (from == from_end) { from=page; from_end=page_end; } - length= (uint) ((*key++ = *from++)) << 8; - if (from == from_end) { from=page; from_end=page_end; } - length+= (uint) ((*key++ = *from++)); + if (from == from_end) { from=page; from_end=page_end; } + length= (uint) ((*key++ = *from++)) << 8; + if (from == from_end) { from=page; from_end=page_end; } + length+= (uint) ((*key++ = *from++)); } } else @@ -1024,7 +1272,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, if ((tmp=(uint) (from_end-from)) <= length) { - key+=tmp; /* Use old key */ + key+=tmp; /* Use old key */ length-=tmp; from=page; from_end=page_end; } @@ -1035,7 +1283,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, length=keyseg->length+nod_flag; if ((tmp=(uint) (from_end-from)) <= length) { - memcpy(key+tmp,page,length-tmp); /* Get last part of key */ + memcpy(key+tmp,page,length-tmp); /* Get last part of key */ *page_pos= page+length-tmp; } else @@ -1044,7 +1292,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, { DBUG_PRINT("error",("Error when unpacking key")); my_errno=HA_ERR_CRASHED; - return 0; /* Error */ + return 0; /* Error */ } memcpy((byte*) key,(byte*) from,(size_t) length); *page_pos= from+length; @@ -1053,11 +1301,11 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, } - /* Get key at position without knowledge of previous key */ - /* Returns pointer to next key */ + /* Get key at position without knowledge of previous key */ + /* Returns pointer to next key */ uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, - uchar *key, uchar *keypos, uint *return_key_length) + uchar *key, uchar *keypos, uint *return_key_length) { uint nod_flag; DBUG_ENTER("_mi_get_key"); @@ -1071,14 +1319,14 @@ uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, else { page+=2+nod_flag; - key[0]=0; /* safety */ + key[0]=0; /* safety */ while (page <= keypos) { *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); if (*return_key_length == 0) { - my_errno=HA_ERR_CRASHED; - DBUG_RETURN(0); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); } } } @@ -1087,12 +1335,12 @@ uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, } /* _mi_get_key */ - /* Get key at position without knowledge of previous key */ - /* Returns 0 if ok */ + /* Get key at position without knowledge of previous key */ + /* Returns 0 if ok */ static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, - uchar *key, uchar *keypos, - uint *return_key_length) + uchar *key, uchar *keypos, + uint *return_key_length) { uint nod_flag; DBUG_ENTER("_mi_get_prev_key"); @@ -1102,20 +1350,20 @@ static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, { *return_key_length=keyinfo->keylength; bmove((byte*) key,(byte*) keypos- *return_key_length-nod_flag, - *return_key_length); + *return_key_length); DBUG_RETURN(0); } else { page+=2+nod_flag; - key[0]=0; /* safety */ + key[0]=0; /* safety */ while (page < keypos) { *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); if (*return_key_length == 0) { - my_errno=HA_ERR_CRASHED; - DBUG_RETURN(1); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(1); } } } @@ -1124,11 +1372,11 @@ static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, - /* Get last key from key-page */ - /* Return pointer to where key starts */ + /* Get last key from key-page */ + /* Return pointer to where key starts */ uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, - uchar *lastkey, uchar *endpos, uint *return_key_length) + uchar *lastkey, uchar *endpos, uint *return_key_length) { uint nod_flag; uchar *lastpos; @@ -1153,9 +1401,9 @@ uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,lastkey); if (*return_key_length == 0) { - DBUG_PRINT("error",("Couldn't find last key: page: %lx",page)); - my_errno=HA_ERR_CRASHED; - DBUG_RETURN(0); + DBUG_PRINT("error",("Couldn't find last key: page: %lx",page)); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); } } } @@ -1164,7 +1412,7 @@ uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, } /* _mi_get_last_key */ - /* Calculate length of key */ + /* Calculate length of key */ uint _mi_keylength(MI_KEYDEF *keyinfo, register uchar *key) { @@ -1179,7 +1427,7 @@ uint _mi_keylength(MI_KEYDEF *keyinfo, register uchar *key) { if (keyseg->flag & HA_NULL_PART) if (!*key++) - continue; + continue; if (keyseg->flag & (HA_SPACE_PACK | HA_BLOB_PART | HA_VAR_LENGTH)) { uint length; @@ -1193,28 +1441,28 @@ uint _mi_keylength(MI_KEYDEF *keyinfo, register uchar *key) } /* _mi_keylength */ - /* Move a key */ + /* Move a key */ uchar *_mi_move_key(MI_KEYDEF *keyinfo, uchar *to, uchar *from) { reg1 uint length; memcpy((byte*) to, (byte*) from, - (size_t) (length=_mi_keylength(keyinfo,from))); + (size_t) (length=_mi_keylength(keyinfo,from))); return to+length; } - /* Find next/previous record with same key */ - /* This can't be used when database is touched after last read */ + /* Find next/previous record with same key */ + /* This can't be used when database is touched after last read */ int _mi_search_next(register MI_INFO *info, register MI_KEYDEF *keyinfo, - uchar *key, uint key_length, uint nextflag, my_off_t pos) + uchar *key, uint key_length, uint nextflag, my_off_t pos) { int error; uint nod_flag; uchar lastkey[MI_MAX_KEY_BUFF]; DBUG_ENTER("_mi_search_next"); DBUG_PRINT("enter",("nextflag: %d lastpos: %ld int_keypos: %lx", - nextflag,(long) info->lastpos,info->int_keypos)); + nextflag,(long) info->lastpos,info->int_keypos)); DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_length);); /* Force full read if we are at last key or if we are not on a leaf @@ -1228,51 +1476,53 @@ int _mi_search_next(register MI_INFO *info, register MI_KEYDEF *keyinfo, info->page_changed || (info->int_keytree_version != keyinfo->version && (info->int_nod_flag || info->buff_used))) - DBUG_RETURN(_mi_search(info,keyinfo,key,key_length, - nextflag | SEARCH_SAVE_BUFF, pos)); + DBUG_RETURN(_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, pos)); if (info->buff_used) { if (!_mi_fetch_keypage(info,keyinfo,info->last_search_keypage, - info->buff,0)) + info->buff,0)) DBUG_RETURN(-1); info->buff_used=0; } /* Last used buffer is in info->buff */ nod_flag=mi_test_if_nod(info->buff); - memcpy(lastkey,key,key_length); - if (nextflag & SEARCH_BIGGER) /* Next key */ + if (nextflag & SEARCH_BIGGER) /* Next key */ { my_off_t tmp_pos=_mi_kpos(nod_flag,info->int_keypos); if (tmp_pos != HA_OFFSET_ERROR) { - if ((error=_mi_search(info,keyinfo,key,key_length, - nextflag | SEARCH_SAVE_BUFF, tmp_pos)) <=0) - DBUG_RETURN(error); + if ((error=_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, tmp_pos)) <=0) + DBUG_RETURN(error); } + memcpy(lastkey,key,key_length); if (!(info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag, - &info->int_keypos,lastkey))) + &info->int_keypos,lastkey))) DBUG_RETURN(-1); } - else /* Previous key */ + else /* Previous key */ { uint length; /* Find start of previous key */ info->int_keypos=_mi_get_last_key(info,keyinfo,info->buff,lastkey, - info->int_keypos, &length); + info->int_keypos, &length); if (!info->int_keypos) DBUG_RETURN(-1); if (info->int_keypos == info->buff+2) - DBUG_RETURN(_mi_search(info,keyinfo,key,key_length, - nextflag | SEARCH_SAVE_BUFF, pos)); - if ((error=_mi_search(info,keyinfo,key,0,nextflag | SEARCH_SAVE_BUFF, - _mi_kpos(nod_flag,info->int_keypos))) <= 0) + DBUG_RETURN(_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, pos)); + if ((error=_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, + _mi_kpos(nod_flag,info->int_keypos))) <= 0) DBUG_RETURN(error); + /* QQ: We should be able to optimize away the following call */ if (! _mi_get_last_key(info,keyinfo,info->buff,lastkey, - info->int_keypos,&info->lastkey_length)) + info->int_keypos,&info->lastkey_length)) DBUG_RETURN(-1); } memcpy(info->lastkey,lastkey,info->lastkey_length); @@ -1282,11 +1532,11 @@ int _mi_search_next(register MI_INFO *info, register MI_KEYDEF *keyinfo, } /* _mi_search_next */ - /* Search after position for the first row in an index */ - /* This is stored in info->lastpos */ + /* Search after position for the first row in an index */ + /* This is stored in info->lastpos */ int _mi_search_first(register MI_INFO *info, register MI_KEYDEF *keyinfo, - register my_off_t pos) + register my_off_t pos) { uint nod_flag; uchar *page; @@ -1311,7 +1561,7 @@ int _mi_search_first(register MI_INFO *info, register MI_KEYDEF *keyinfo, } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page, - info->lastkey); + info->lastkey); info->int_keypos=page; info->int_maxpos=info->buff+mi_getint(info->buff)-1; info->int_nod_flag=nod_flag; info->int_keytree_version=keyinfo->version; @@ -1324,11 +1574,11 @@ int _mi_search_first(register MI_INFO *info, register MI_KEYDEF *keyinfo, } /* _mi_search_first */ - /* Search after position for the last row in an index */ - /* This is stored in info->lastpos */ + /* Search after position for the last row in an index */ + /* This is stored in info->lastpos */ int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, - register my_off_t pos) + register my_off_t pos) { uint nod_flag; uchar *buff,*page; @@ -1336,7 +1586,7 @@ int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, if (pos == HA_OFFSET_ERROR) { - my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ info->lastpos= HA_OFFSET_ERROR; DBUG_RETURN(-1); } @@ -1354,7 +1604,7 @@ int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); if (!_mi_get_last_key(info,keyinfo,buff,info->lastkey,page, - &info->lastkey_length)) + &info->lastkey_length)) DBUG_RETURN(-1); info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); info->int_keypos=info->int_maxpos=page; @@ -1374,22 +1624,22 @@ int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, ** Functions to store and pack a key in a page ** ** mi_calc_xx_key_length takes the following arguments: -** nod_flag If nod: Length of nod-pointer -** next_key Position to pos after the new key in buffer -** org_key Key that was before the next key in buffer -** prev_key Last key before current key -** key Key that will be stored -** s_temp Information how next key will be packed +** nod_flag If nod: Length of nod-pointer +** next_key Position to pos after the new key in buffer +** org_key Key that was before the next key in buffer +** prev_key Last key before current key +** key Key that will be stored +** s_temp Information how next key will be packed ****************************************************************************/ /* Static length key */ int _mi_calc_static_key_length(MI_KEYDEF *keyinfo,uint nod_flag, - uchar *next_pos __attribute__((unused)), - uchar *org_key __attribute__((unused)), - uchar *prev_key __attribute__((unused)), - uchar *key, MI_KEY_PARAM *s_temp) + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) { s_temp->key=key; return (int) (s_temp->totlength=keyinfo->keylength+nod_flag); @@ -1399,10 +1649,10 @@ _mi_calc_static_key_length(MI_KEYDEF *keyinfo,uint nod_flag, int _mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, - uchar *next_pos __attribute__((unused)), - uchar *org_key __attribute__((unused)), - uchar *prev_key __attribute__((unused)), - uchar *key, MI_KEY_PARAM *s_temp) + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) { s_temp->key=key; return (int) (s_temp->totlength=_mi_keylength(keyinfo,key)+nod_flag); @@ -1417,10 +1667,10 @@ _mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, If the max length of first key segment <= 127 characters the prefix is 1 byte else it's 2 byte - prefix byte The high bit is set if this is a prefix for the prev key - length Packed length if the previous was a prefix byte - [length] Length character of data - next-key-seg Next key segments + prefix byte The high bit is set if this is a prefix for the prev key + length Packed length if the previous was a prefix byte + [length] Length character of data + next-key-seg Next key segments If the first segment can have NULL: The length is 0 for NULLS and 1+length for not null columns. @@ -1429,8 +1679,8 @@ _mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, int _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, - uchar *org_key, uchar *prev_key, uchar *key, - MI_KEY_PARAM *s_temp) + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) { reg1 MI_KEYSEG *keyseg; int length; @@ -1471,15 +1721,15 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, s_temp->key=key; s_temp->ref_length=s_temp->key_length=0; s_temp->totlength=key_length-1+diff_flag; - s_temp->next_key_pos=0; /* No next key */ + s_temp->next_key_pos=0; /* No next key */ return (s_temp->totlength); } s_temp->store_not_null=1; - key_length--; /* We don't store NULL */ + key_length--; /* We don't store NULL */ if (prev_key && !*prev_key++) - org_key=prev_key=0; /* Can't pack against prev */ + org_key=prev_key=0; /* Can't pack against prev */ else if (org_key) - org_key++; /* Skipp NULL */ + org_key++; /* Skipp NULL */ } else s_temp->store_not_null=0; @@ -1495,14 +1745,14 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, if (prev_key) { get_key_length(org_key_length,prev_key); - s_temp->prev_key=prev_key; /* Pointer at data */ + s_temp->prev_key=prev_key; /* Pointer at data */ /* Don't use key-pack if length == 0 */ if (new_key_length && new_key_length == org_key_length) same_length=1; else if (new_key_length > org_key_length) end=key + org_key_length; - if (sort_order) /* SerG */ + if (sort_order) /* SerG */ { while (key < end && sort_order[*key] == sort_order[*prev_key]) { @@ -1513,7 +1763,7 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, { while (key < end && *key == *prev_key) { - key++; prev_key++; + key++; prev_key++; } } } @@ -1528,15 +1778,15 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, length=(int) key_length-(int) (key_end-start)-length_pack; length+= diff_flag; if (next_key) - { /* Can't combine with next */ - s_temp->n_length= *next_key; /* Needed by _mi_store_key */ + { /* Can't combine with next */ + s_temp->n_length= *next_key; /* Needed by _mi_store_key */ next_key=0; } } else { if (start != key) - { /* Starts as prev key */ + { /* Starts as prev key */ ref_length= (uint) (key-start); s_temp->ref_length= ref_length + pack_marker; length= (int) (key_length - ref_length); @@ -1547,16 +1797,16 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, } else { - s_temp->key_length+=s_temp->store_not_null; /* If null */ + s_temp->key_length+=s_temp->store_not_null; /* If null */ length= key_length - length_pack+ diff_flag; } } s_temp->totlength=(uint) length; s_temp->prev_length=0; DBUG_PRINT("test",("tot_length: %d length: %d uniq_key_length: %d", - key_length,length,s_temp->key_length)); + key_length,length,s_temp->key_length)); - /* If something after that hasn't length=0, test if we can combine */ + /* If something after that hasn't length=0, test if we can combine */ if ((s_temp->next_key_pos=next_key)) { uint packed,n_length; @@ -1572,134 +1822,134 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, if (!packed) n_length-= s_temp->store_not_null; - if (n_length || packed) /* Don't pack 0 length keys */ + if (n_length || packed) /* Don't pack 0 length keys */ { uint next_length_pack, new_ref_length=s_temp->ref_length; if (packed) { - /* If first key and next key is packed (only on delete) */ - if (!prev_key && org_key) - { - get_key_length(org_key_length,org_key); - key=start; - if (sort_order) /* SerG */ - { - while (key < end && sort_order[*key] == sort_order[*org_key]) - { - key++; org_key++; - } - } - else - { - while (key < end && *key == *org_key) - { - key++; org_key++; - } - } - if ((new_ref_length= (uint) (key - start))) - new_ref_length+=pack_marker; - } - - if (!n_length) - { - /* - We put a different key between two identical variable length keys - Extend next key to have same prefix as this key - */ - if (new_ref_length) /* prefix of previus key */ - { /* make next key longer */ - s_temp->part_of_prev_key= new_ref_length; - s_temp->prev_length= org_key_length - - (new_ref_length-pack_marker); - s_temp->n_ref_length= s_temp->n_length= s_temp->prev_length; - n_length= get_pack_length(s_temp->prev_length); - s_temp->prev_key+= (new_ref_length - pack_marker); - length+= s_temp->prev_length + n_length; - } - else - { /* Can't use prev key */ - s_temp->part_of_prev_key=0; - s_temp->prev_length= org_key_length; - s_temp->n_ref_length=s_temp->n_length= org_key_length; - length+= org_key_length; - /* +get_pack_length(org_key_length); */ - } - return (int) length; - } - - ref_length=n_length; - get_key_pack_length(n_length,next_length_pack,next_key); - - /* Test if new keys has fewer characters that match the previous key */ - if (!new_ref_length) - { /* Can't use prev key */ - s_temp->part_of_prev_key= 0; - s_temp->prev_length= ref_length; - s_temp->n_ref_length= s_temp->n_length= n_length+ref_length; - /* s_temp->prev_key+= get_pack_length(org_key_length); */ - return (int) length+ref_length-next_length_pack; - } - if (ref_length+pack_marker > new_ref_length) - { - uint new_pack_length=new_ref_length-pack_marker; - /* We must copy characters from the original key to the next key */ - s_temp->part_of_prev_key= new_ref_length; - s_temp->prev_length= ref_length - new_pack_length; - s_temp->n_ref_length=s_temp->n_length=n_length + s_temp->prev_length; - s_temp->prev_key+= new_pack_length; -/* +get_pack_length(org_key_length); */ - length= length-get_pack_length(ref_length)+ - get_pack_length(new_pack_length); - return (int) length + s_temp->prev_length; - } + /* If first key and next key is packed (only on delete) */ + if (!prev_key && org_key) + { + get_key_length(org_key_length,org_key); + key=start; + if (sort_order) /* SerG */ + { + while (key < end && sort_order[*key] == sort_order[*org_key]) + { + key++; org_key++; + } + } + else + { + while (key < end && *key == *org_key) + { + key++; org_key++; + } + } + if ((new_ref_length= (uint) (key - start))) + new_ref_length+=pack_marker; + } + + if (!n_length) + { + /* + We put a different key between two identical variable length keys + Extend next key to have same prefix as this key + */ + if (new_ref_length) /* prefix of previus key */ + { /* make next key longer */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= org_key_length - + (new_ref_length-pack_marker); + s_temp->n_ref_length= s_temp->n_length= s_temp->prev_length; + n_length= get_pack_length(s_temp->prev_length); + s_temp->prev_key+= (new_ref_length - pack_marker); + length+= s_temp->prev_length + n_length; + } + else + { /* Can't use prev key */ + s_temp->part_of_prev_key=0; + s_temp->prev_length= org_key_length; + s_temp->n_ref_length=s_temp->n_length= org_key_length; + length+= org_key_length; + /* +get_pack_length(org_key_length); */ + } + return (int) length; + } + + ref_length=n_length; + get_key_pack_length(n_length,next_length_pack,next_key); + + /* Test if new keys has fewer characters that match the previous key */ + if (!new_ref_length) + { /* Can't use prev key */ + s_temp->part_of_prev_key= 0; + s_temp->prev_length= ref_length; + s_temp->n_ref_length= s_temp->n_length= n_length+ref_length; + /* s_temp->prev_key+= get_pack_length(org_key_length); */ + return (int) length+ref_length-next_length_pack; + } + if (ref_length+pack_marker > new_ref_length) + { + uint new_pack_length=new_ref_length-pack_marker; + /* We must copy characters from the original key to the next key */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= ref_length - new_pack_length; + s_temp->n_ref_length=s_temp->n_length=n_length + s_temp->prev_length; + s_temp->prev_key+= new_pack_length; +/* +get_pack_length(org_key_length); */ + length= length-get_pack_length(ref_length)+ + get_pack_length(new_pack_length); + return (int) length + s_temp->prev_length; + } } else { - /* Next key wasn't a prefix of previous key */ - ref_length=0; - next_length_pack=0; + /* Next key wasn't a prefix of previous key */ + ref_length=0; + next_length_pack=0; } DBUG_PRINT("test",("length: %d next_key: %lx",length,next_key)); { - uint tmp_length; - key=(start+=ref_length); - if (key+n_length < key_end) /* Normalize length based */ - key_end=key+n_length; - if (sort_order) /* SerG */ - { + uint tmp_length; + key=(start+=ref_length); + if (key+n_length < key_end) /* Normalize length based */ + key_end=key+n_length; + if (sort_order) /* SerG */ + { while (key < key_end && sort_order[*key] == - sort_order[*next_key]) - { - key++; next_key++; - } - } - else - { - while (key < key_end && *key == *next_key) - { - key++; next_key++; - } - } - if (!(tmp_length=(uint) (key-start))) - { /* Key can't be re-packed */ - s_temp->next_key_pos=0; - return length; - } - ref_length+=tmp_length; - n_length-=tmp_length; - length-=tmp_length+next_length_pack; /* We gained these chars */ + sort_order[*next_key]) + { + key++; next_key++; + } + } + else + { + while (key < key_end && *key == *next_key) + { + key++; next_key++; + } + } + if (!(tmp_length=(uint) (key-start))) + { /* Key can't be re-packed */ + s_temp->next_key_pos=0; + return length; + } + ref_length+=tmp_length; + n_length-=tmp_length; + length-=tmp_length+next_length_pack; /* We gained these chars */ } if (n_length == 0) { - s_temp->n_ref_length=pack_marker; /* Same as prev key */ + s_temp->n_ref_length=pack_marker; /* Same as prev key */ } else { - s_temp->n_ref_length=ref_length | pack_marker; - length+= get_pack_length(n_length); - s_temp->n_length=n_length; + s_temp->n_ref_length=ref_length | pack_marker; + length+= get_pack_length(n_length); + s_temp->n_length=n_length; } } } @@ -1711,15 +1961,15 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, int _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, - uchar *org_key, uchar *prev_key, uchar *key, - MI_KEY_PARAM *s_temp) + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) { uint length,key_length,ref_length; s_temp->totlength=key_length=_mi_keylength(keyinfo,key)+nod_flag; s_temp->key=key; s_temp->prev_key=org_key; - if (prev_key) /* If not first key in block */ + if (prev_key) /* If not first key in block */ { /* pack key against previous key */ /* @@ -1738,7 +1988,7 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, s_temp->ref_length=ref_length=0; length=key_length+1; } - if ((s_temp->next_key_pos=next_key)) /* If another key after */ + if ((s_temp->next_key_pos=next_key)) /* If another key after */ { /* pack key against next key */ uint next_length,next_length_pack; @@ -1749,21 +1999,21 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, { uchar *end; for (key= s_temp->key, end=key+next_length ; - *key == *org_key && key < end; - key++,org_key++) ; + *key == *org_key && key < end; + key++,org_key++) ; ref_length= (uint) (key - s_temp->key); } if (next_length > ref_length) { /* We put a key with different case between two keys with the same prefix - Extend next key to have same prefix as - this key */ + Extend next key to have same prefix as + this key */ s_temp->n_ref_length= ref_length; s_temp->prev_length= next_length-ref_length; s_temp->prev_key+= ref_length; return (int) (length+ s_temp->prev_length - next_length_pack + - get_pack_length(ref_length)); + get_pack_length(ref_length)); } /* Check how many characters are identical to next key */ key= s_temp->key+next_length; @@ -1771,12 +2021,12 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, if ((ref_length= (uint) (key - s_temp->key)-1) == next_length) { s_temp->next_key_pos=0; - return length; /* can't pack next key */ + return length; /* can't pack next key */ } s_temp->prev_length=0; s_temp->n_ref_length=ref_length; return (int) (length-(ref_length - next_length) - next_length_pack + - get_pack_length(ref_length)); + get_pack_length(ref_length)); } return (int) length; } @@ -1789,8 +2039,8 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, /* store key without compression */ void _mi_store_static_key(MI_KEYDEF *keyinfo __attribute__((unused)), - register uchar *key_pos, - register MI_KEY_PARAM *s_temp) + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) { memcpy((byte*) key_pos,(byte*) s_temp->key,(size_t) s_temp->totlength); } @@ -1804,8 +2054,8 @@ void _mi_store_static_key(MI_KEYDEF *keyinfo __attribute__((unused)), void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), - register uchar *key_pos, - register MI_KEY_PARAM *s_temp) + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) { uint length; uchar *start; @@ -1826,9 +2076,9 @@ void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->key_length); } bmove((byte*) key_pos,(byte*) s_temp->key, - (length=s_temp->totlength-(uint) (key_pos-start))); + (length=s_temp->totlength-(uint) (key_pos-start))); - if (!s_temp->next_key_pos) /* No following key */ + if (!s_temp->next_key_pos) /* No following key */ return; key_pos+=length; @@ -1838,14 +2088,14 @@ void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), if (s_temp->part_of_prev_key) { store_pack_length(s_temp->pack_marker == 128,key_pos, - s_temp->part_of_prev_key); + s_temp->part_of_prev_key); store_key_length_inc(key_pos,s_temp->n_length); } else { s_temp->n_length+= s_temp->store_not_null; store_pack_length(s_temp->pack_marker == 128,key_pos, - s_temp->n_length); + s_temp->n_length); } memcpy(key_pos, s_temp->prev_key, s_temp->prev_length); } @@ -1853,7 +2103,7 @@ void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), { store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_ref_length); if (s_temp->n_ref_length == s_temp->pack_marker) - return; /* Identical key */ + return; /* Identical key */ store_key_length(key_pos,s_temp->n_length); } else @@ -1867,18 +2117,18 @@ void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), /* variable length key with prefix compression */ void _mi_store_bin_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), - register uchar *key_pos, - register MI_KEY_PARAM *s_temp) + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) { store_key_length_inc(key_pos,s_temp->ref_length); memcpy((char*) key_pos,(char*) s_temp->key+s_temp->ref_length, - (size_t) s_temp->totlength-s_temp->ref_length); + (size_t) s_temp->totlength-s_temp->ref_length); if (s_temp->next_key_pos) { key_pos+=(uint) (s_temp->totlength-s_temp->ref_length); store_key_length_inc(key_pos,s_temp->n_ref_length); - if (s_temp->prev_length) /* If we must extend key */ + if (s_temp->prev_length) /* If we must extend key */ { memcpy(key_pos,s_temp->prev_key,s_temp->prev_length); } diff --git a/myisam/mi_static.c b/myisam/mi_static.c index f790f90ca78..37b9ac04b7a 100644 --- a/myisam/mi_static.c +++ b/myisam/mi_static.c @@ -1,21 +1,21 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* - Static variables for pisam library. All definied here for easy making of + Static variables for MyISAM library. All definied here for easy making of a shared library */ @@ -32,7 +32,7 @@ my_string myisam_log_filename=(char*) "myisam.log"; File myisam_log_file= -1; uint myisam_quick_table_bits=9; uint myisam_block_size=MI_KEY_BLOCK_LENGTH; /* Best by test */ -my_bool myisam_flush=0,myisam_delay_key_write=0; +my_bool myisam_flush=0, myisam_delay_key_write=0, myisam_single_user=0; #if defined(THREAD) && !defined(DONT_USE_RW_LOCKS) my_bool myisam_concurrent_insert=1; #else @@ -40,10 +40,12 @@ my_bool myisam_concurrent_insert=0; #endif my_off_t myisam_max_extra_temp_length= MI_MAX_TEMP_LENGTH; my_off_t myisam_max_temp_length= MAX_FILE_SIZE; +ulong myisam_bulk_insert_tree_size=8192*1024; - -/* read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ */ -/* Position is , == , >= , <= , > , < */ +/* + read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ + Position is , == , >= , <= , > , < +*/ uint NEAR myisam_read_vec[]= { diff --git a/myisam/mi_statrec.c b/myisam/mi_statrec.c index 05ff40d8921..9afebce2ae6 100644 --- a/myisam/mi_statrec.c +++ b/myisam/mi_statrec.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -27,17 +27,16 @@ int _mi_write_static_record(MI_INFO *info, const byte *record) { my_off_t filepos=info->s->state.dellink; info->rec_cache.seek_not_done=1; /* We have done a seek */ - VOID(my_seek(info->dfile,info->s->state.dellink+1,MY_SEEK_SET,MYF(0))); - - if (my_read(info->dfile,(char*) &temp[0],info->s->base.rec_reflength, - MYF(MY_NABP))) + if (my_pread(info->dfile,(char*) &temp[0],info->s->base.rec_reflength, + info->s->state.dellink+1, + MYF(MY_NABP))) goto err; info->s->state.dellink= _mi_rec_pos(info->s,temp); info->state->del--; info->state->empty-=info->s->base.pack_reclength; - VOID(my_seek(info->dfile,filepos,MY_SEEK_SET,MYF(0))); - if (my_write(info->dfile, (char*) record, info->s->base.reclength, - MYF(MY_NABP))) + if (my_pwrite(info->dfile, (char*) record, info->s->base.reclength, + filepos, + MYF(MY_NABP))) goto err; } else @@ -64,16 +63,18 @@ int _mi_write_static_record(MI_INFO *info, const byte *record) else { info->rec_cache.seek_not_done=1; /* We have done a seek */ - VOID(my_seek(info->dfile,info->state->data_file_length, - MY_SEEK_SET,MYF(0))); - if (my_write(info->dfile,(char*) record,info->s->base.reclength, - info->s->write_flag)) + if (my_pwrite(info->dfile,(char*) record,info->s->base.reclength, + info->state->data_file_length, + info->s->write_flag)) goto err; if (info->s->base.pack_reclength != info->s->base.reclength) { uint length=info->s->base.pack_reclength - info->s->base.reclength; bzero((char*) temp,length); - if (my_write(info->dfile, (byte*) temp,length, info->s->write_flag)) + if (my_pwrite(info->dfile, (byte*) temp,length, + info->state->data_file_length+ + info->s->base.reclength, + info->s->write_flag)) goto err; } } @@ -88,9 +89,10 @@ int _mi_write_static_record(MI_INFO *info, const byte *record) int _mi_update_static_record(MI_INFO *info, my_off_t pos, const byte *record) { info->rec_cache.seek_not_done=1; /* We have done a seek */ - VOID(my_seek(info->dfile,pos,MY_SEEK_SET,MYF(0))); - return (my_write(info->dfile,(char*) record,info->s->base.reclength, - MYF(MY_NABP)) != 0); + return (my_pwrite(info->dfile, + (char*) record,info->s->base.reclength, + pos, + MYF(MY_NABP)) != 0); } @@ -104,9 +106,8 @@ int _mi_delete_static_record(MI_INFO *info) _mi_dpointer(info,temp+1,info->s->state.dellink); info->s->state.dellink = info->lastpos; info->rec_cache.seek_not_done=1; - VOID(my_seek(info->dfile,info->lastpos,MY_SEEK_SET,MYF(0))); - return (my_write(info->dfile,(byte*) temp, 1+info->s->rec_reflength, - MYF(MY_NABP)) != 0); + return (my_pwrite(info->dfile,(byte*) temp, 1+info->s->rec_reflength, + info->lastpos, MYF(MY_NABP)) != 0); } @@ -129,9 +130,9 @@ int _mi_cmp_static_record(register MI_INFO *info, register const byte *old) if ((info->opt_flag & READ_CHECK_USED)) { /* If check isn't disabled */ info->rec_cache.seek_not_done=1; /* We have done a seek */ - VOID(my_seek(info->dfile,info->lastpos,MY_SEEK_SET,MYF(0))); - if (my_read(info->dfile, (char*) info->rec_buff, info->s->base.reclength, - MYF(MY_NABP))) + if (my_pread(info->dfile, (char*) info->rec_buff, info->s->base.reclength, + info->lastpos, + MYF(MY_NABP))) DBUG_RETURN(-1); if (memcmp((byte*) info->rec_buff, (byte*) old, (uint) info->s->base.reclength)) @@ -152,9 +153,8 @@ int _mi_cmp_static_unique(MI_INFO *info, MI_UNIQUEDEF *def, DBUG_ENTER("_mi_cmp_static_unique"); info->rec_cache.seek_not_done=1; /* We have done a seek */ - VOID(my_seek(info->dfile,pos,MY_SEEK_SET,MYF(0))); - if (my_read(info->dfile, (char*) info->rec_buff, info->s->base.reclength, - MYF(MY_NABP))) + if (my_pread(info->dfile, (char*) info->rec_buff, info->s->base.reclength, + pos, MYF(MY_NABP))) DBUG_RETURN(-1); DBUG_RETURN(mi_unique_comp(def, record, info->rec_buff, def->null_are_equal)); @@ -181,8 +181,7 @@ int _mi_read_static_record(register MI_INFO *info, register my_off_t pos, error=my_pread(info->dfile,(char*) record,info->s->base.reclength, pos,MYF(MY_NABP)) != 0; - if (info->s->r_locks == 0 && info->s->w_locks == 0) - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); if (! error) { if (!*record) @@ -195,7 +194,7 @@ int _mi_read_static_record(register MI_INFO *info, register my_off_t pos, } return(-1); /* Error on read */ } - VOID(_mi_writeinfo(info,0)); /* No such record */ + fast_mi_writeinfo(info); /* No such record */ return(-1); } @@ -222,7 +221,7 @@ int _mi_read_rnd_static_record(MI_INFO *info, byte *buf, (skipp_deleted_blocks || !filepos)) { cache_read=1; /* Read record using cache */ - cache_length=(uint) (info->rec_cache.rc_end - info->rec_cache.rc_pos); + cache_length=(uint) (info->rec_cache.read_end - info->rec_cache.read_pos); } else info->rec_cache.seek_not_done=1; /* Filepos is changed */ @@ -240,7 +239,7 @@ int _mi_read_rnd_static_record(MI_INFO *info, byte *buf, { /* We don't nead new info */ #ifndef UNSAFE_LOCKING if ((! cache_read || share->base.reclength > cache_length) && - share->r_locks == 0 && share->w_locks == 0) + share->tot_locks == 0) { /* record not in cache */ if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) @@ -257,7 +256,7 @@ int _mi_read_rnd_static_record(MI_INFO *info, byte *buf, DBUG_PRINT("test",("filepos: %ld (%ld) records: %ld del: %ld", filepos/share->base.reclength,filepos, info->state->records, info->state->del)); - VOID(_mi_writeinfo(info,0)); + fast_mi_writeinfo(info); DBUG_RETURN(my_errno=HA_ERR_END_OF_FILE); } info->lastpos= filepos; diff --git a/myisam/mi_test1.c b/myisam/mi_test1.c index ae09bd4142e..8ea97c8e489 100644 --- a/myisam/mi_test1.c +++ b/myisam/mi_test1.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -17,20 +17,23 @@ /* Testing of the basic functions of a MyISAM table */ #include "myisam.h" -#include <getopt.h> +#include <my_getopt.h> #include <m_string.h> #define MAX_REC_LENGTH 1024 -static int rec_pointer_size=0,verbose=0,flags[50]; -static int key_field=FIELD_SKIPP_PRESPACE,extra_field=FIELD_SKIPP_ENDSPACE; +static void usage(); + +static int rec_pointer_size=0, flags[50]; +static int key_field=FIELD_SKIP_PRESPACE,extra_field=FIELD_SKIP_ENDSPACE; static int key_type=HA_KEYTYPE_NUM; static int create_flag=0; -static uint insert_count= 1000,update_count=1000,remove_count=1000; -static uint pack_keys=0,pack_seg=0,null_fields=0,key_length=6,skip_update=0; -static uint unique_key=HA_NOSAME,key_cacheing=0,opt_unique=0; -static uint silent; +static uint insert_count, update_count, remove_count; +static uint pack_keys=0, pack_seg=0, key_length; +static uint unique_key=HA_NOSAME; +static my_bool key_cacheing, null_fields, silent, skip_update, opt_unique, + verbose; static MI_COLUMNDEF recinfo[4]; static MI_KEYDEF keyinfo[10]; static MI_KEYSEG keyseg[10]; @@ -47,14 +50,14 @@ int main(int argc,char *argv[]) MY_INIT(argv[0]); my_init(); if (key_cacheing) - init_key_cache(IO_SIZE*16,(uint) IO_SIZE*4*10); + init_key_cache(IO_SIZE*16); get_options(argc,argv); exit(run_test("test1")); } -int run_test(const char *filename) +static int run_test(const char *filename) { MI_INFO *file; int i,j,error,deleted,rec_length,uniques=0; @@ -501,139 +504,155 @@ static void update_record(char *record) } -static struct option long_options[] = +static struct my_option my_long_options[] = { - {"checksum", no_argument, 0, 'c'}, + {"checksum", 'c', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF - {"debug", required_argument, 0, '#'}, + {"debug", '#', "Undocumented", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #endif - {"delete_rows", required_argument, 0, 'd'}, - {"help", no_argument, 0, '?'}, - {"insert_rows", required_argument, 0, 'i'}, - {"key_alpha", no_argument, 0, 'a'}, - {"key_binary_pack", no_argument, 0, 'B'}, - {"key_blob", required_argument, 0, 'b'}, - {"key_cache", no_argument, 0, 'K'}, - {"key_length", required_argument, 0, 'k'}, - {"key_multiple", no_argument, 0, 'm'}, - {"key_prefix_pack", no_argument, 0, 'P'}, - {"key_space_pack", no_argument, 0, 'p'}, - {"key_varchar", no_argument, 0, 'w'}, - {"null_fields", no_argument, 0, 'N'}, - {"row_fixed_size", no_argument, 0, 'S'}, - {"row_pointer_size", required_argument, 0, 'R'}, - {"silent", no_argument, 0, 's'}, - {"skip_update", no_argument, 0, 'U'}, - {"unique", no_argument, 0, 'C'}, - {"update_rows", required_argument, 0, 'u'}, - {"verbose", no_argument, 0, 'v'}, - {"version", no_argument, 0, 'V'}, - {0, 0, 0, 0} + {"delete_rows", 'd', "Undocumented", (gptr*) &remove_count, + (gptr*) &remove_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"help", '?', "Display help and exit", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"insert_rows", 'i', "Undocumented", (gptr*) &insert_count, + (gptr*) &insert_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"key_alpha", 'a', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_binary_pack", 'B', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_blob", 'b', "Undocumented", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"key_cache", 'K', "Undocumented", (gptr*) &key_cacheing, + (gptr*) &key_cacheing, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_length", 'k', "Undocumented", (gptr*) &key_length, (gptr*) &key_length, + 0, GET_UINT, REQUIRED_ARG, 6, 0, 0, 0, 0, 0}, + {"key_multiple", 'm', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_prefix_pack", 'P', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_space_pack", 'p', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_varchar", 'w', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"null_fields", 'N', "Undocumented", + (gptr*) &null_fields, (gptr*) &null_fields, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + {"row_fixed_size", 'S', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"row_pointer_size", 'R', "Undocumented", (gptr*) &rec_pointer_size, + (gptr*) &rec_pointer_size, 0, GET_INT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', "Undocumented", + (gptr*) &silent, (gptr*) &silent, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"skip_update", 'U', "Undocumented", (gptr*) &skip_update, + (gptr*) &skip_update, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"unique", 'C', "Undocumented", (gptr*) &opt_unique, (gptr*) &opt_unique, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"update_rows", 'u', "Undocumented", (gptr*) &update_count, + (gptr*) &update_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Be more verbose", (gptr*) &verbose, (gptr*) &verbose, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', "Print version number and exit", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; +static my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case 'a': + key_type= HA_KEYTYPE_TEXT; + break; + case 'c': + create_flag|= HA_CREATE_CHECKSUM; + break; + case 'R': /* Length of record pointer */ + if (rec_pointer_size > 3) + rec_pointer_size=0; + break; + case 'P': + pack_keys= HA_PACK_KEY; /* Use prefix compression */ + break; + case 'B': + pack_keys= HA_BINARY_PACK_KEY; /* Use binary compression */ + break; + case 'S': + if (key_field == FIELD_VARCHAR) + { + create_flag=0; /* Static sized varchar */ + } + else if (key_field != FIELD_BLOB) + { + key_field=FIELD_NORMAL; /* static-size record */ + extra_field=FIELD_NORMAL; + } + break; + case 'p': + pack_keys=HA_PACK_KEY; /* Use prefix + space packing */ + pack_seg=HA_SPACE_PACK; + key_type=HA_KEYTYPE_TEXT; + break; + case 'm': + unique_key=0; + break; + case 'b': + key_field=FIELD_BLOB; /* blob key */ + extra_field= FIELD_BLOB; + pack_seg|= HA_BLOB_PART; + key_type= HA_KEYTYPE_VARTEXT; + break; + case 'k': + if (key_length < 4 || key_length > MI_MAX_KEY_LENGTH) + { + fprintf(stderr,"Wrong key length\n"); + exit(1); + } + break; + case 'w': + key_field=FIELD_VARCHAR; /* varchar keys */ + extra_field= FIELD_VARCHAR; + key_type= HA_KEYTYPE_VARTEXT; + pack_seg|= HA_VAR_LENGTH; + create_flag|= HA_PACK_RECORD; + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + break; + case 'V': + printf("test1 Ver 1.2 \n"); + exit(0); + case '#': + DEBUGGER_ON; + DBUG_PUSH (argument); + break; + case '?': + usage(); + exit(1); + } + return 0; +} + + /* Read options */ -static void get_options(int argc,char *argv[]) +static void get_options(int argc, char *argv[]) { - int c,option_index=0; + int ho_error; + + if ((ho_error=handle_options(&argc, &argv, my_long_options, get_one_option))) + exit(ho_error); - while ((c=getopt_long(argc,argv,"abBcCd:i:k:KmPR:SspNu:UvVw#:", - long_options, &option_index)) != EOF) - { - switch(c) { - case 'a': - key_type= HA_KEYTYPE_TEXT; - break; - case 'c': - create_flag|= HA_CREATE_CHECKSUM; - break; - case 'C': - opt_unique=1; - break; - case 'R': /* Length of record pointer */ - rec_pointer_size=atoi(optarg); - if (rec_pointer_size > 3) - rec_pointer_size=0; - break; - case 'P': - pack_keys= HA_PACK_KEY; /* Use prefix compression */ - break; - case 'B': - pack_keys= HA_BINARY_PACK_KEY; /* Use binary compression */ - break; - case 'S': - if (key_field == FIELD_VARCHAR) - { - create_flag=0; /* Static sized varchar */ - } - else if (key_field != FIELD_BLOB) - { - key_field=FIELD_NORMAL; /* static-size record */ - extra_field=FIELD_NORMAL; - } - break; - case 'p': - pack_keys=HA_PACK_KEY; /* Use prefix + space packing */ - pack_seg=HA_SPACE_PACK; - key_type=HA_KEYTYPE_TEXT; - break; - case 'N': - null_fields=1; /* First key part may be null */ - break; - case 'v': /* verbose */ - verbose=1; - break; - case 'd': - remove_count=atoi(optarg); - break; - case 'i': - insert_count=atoi(optarg); - break; - case 'u': - update_count=atoi(optarg); - break; - case 'U': - skip_update=1; - break; - case 'm': - unique_key=0; - break; - case 'b': - key_field=FIELD_BLOB; /* blob key */ - extra_field= FIELD_BLOB; - pack_seg|= HA_BLOB_PART; - key_type= HA_KEYTYPE_VARTEXT; - break; - case 'k': - key_length=atoi(optarg); - if (key_length < 4 || key_length > MI_MAX_KEY_LENGTH) - { - fprintf(stderr,"Wrong key length\n"); - exit(1); - } - break; - case 's': - silent=1; - break; - case 'w': - key_field=FIELD_VARCHAR; /* varchar keys */ - extra_field= FIELD_VARCHAR; - key_type= HA_KEYTYPE_VARTEXT; - pack_seg|= HA_VAR_LENGTH; - create_flag|= HA_PACK_RECORD; - break; - case 'K': /* Use key cacheing */ - key_cacheing=1; - break; - case 'V': - printf("test1 Ver 1.0 \n"); - exit(0); - case '#': - DEBUGGER_ON; - DBUG_PUSH (optarg); - break; - } - } return; } /* get options */ + + +static void usage() +{ + printf("Usage: %s [options]\n\n", my_progname); + my_print_help(my_long_options); + my_print_variables(my_long_options); +} diff --git a/myisam/mi_test2.c b/myisam/mi_test2.c index b66b02afdf9..93538e3ead7 100644 --- a/myisam/mi_test2.c +++ b/myisam/mi_test2.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -148,15 +148,15 @@ int main(int argc, char *argv[]) keyinfo[5].keysegs=1; keyinfo[5].flag = pack_type; - recinfo[0].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[0].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; recinfo[0].length=7; recinfo[0].null_bit=0; recinfo[0].null_pos=0; - recinfo[1].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[1].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; recinfo[1].length=5; recinfo[1].null_bit=0; recinfo[1].null_pos=0; - recinfo[2].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[2].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; recinfo[2].length=9; recinfo[2].null_bit=0; recinfo[2].null_pos=0; @@ -164,11 +164,11 @@ int main(int argc, char *argv[]) recinfo[3].length=STANDARD_LENGTH-7-5-9-4; recinfo[3].null_bit=0; recinfo[3].null_pos=0; - recinfo[4].type=pack_fields ? FIELD_SKIPP_ZERO : 0; + recinfo[4].type=pack_fields ? FIELD_SKIP_ZERO : 0; recinfo[4].length=4; recinfo[4].null_bit=0; recinfo[4].null_pos=0; - recinfo[5].type=pack_fields ? FIELD_SKIPP_ENDSPACE : 0; + recinfo[5].type=pack_fields ? FIELD_SKIP_ENDSPACE : 0; recinfo[5].length=60; recinfo[5].null_bit=0; recinfo[5].null_pos=0; @@ -208,13 +208,13 @@ int main(int argc, char *argv[]) if (!silent) printf("- Writing key:s\n"); if (key_cacheing) - init_key_cache(key_cache_size,(uint) IO_SIZE*4*10); /* Use a small cache */ + init_key_cache(key_cache_size); /* Use a small cache */ if (locking) mi_lock_database(file,F_WRLCK); if (write_cacheing) - mi_extra(file,HA_EXTRA_WRITE_CACHE); + mi_extra(file,HA_EXTRA_WRITE_CACHE,0); if (opt_quick_mode) - mi_extra(file,HA_EXTRA_QUICK); + mi_extra(file,HA_EXTRA_QUICK,0); for (i=0 ; i < recant ; i++) { @@ -261,11 +261,15 @@ int main(int argc, char *argv[]) if (testflag==1) goto end; if (write_cacheing) - if (mi_extra(file,HA_EXTRA_NO_CACHE)) + { + if (mi_extra(file,HA_EXTRA_NO_CACHE,0)) { puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); goto end; } + if (key_cacheing) + resize_key_cache(key_cache_size*2); + } if (!silent) printf("- Delete\n"); @@ -341,70 +345,81 @@ int main(int argc, char *argv[]) } } } - if (testflag==3) goto end; + if (testflag == 3) + goto end; - if (!silent) - printf("- Same key: first - next -> last - prev -> first\n"); - DBUG_PRINT("progpos",("first - next -> last - prev -> first")); for (i=999, dupp_keys=j=0 ; i>0 ; i--) { - if (key1[i] >dupp_keys) { dupp_keys=key1[i]; j=i; } + if (key1[i] > dupp_keys) + { + dupp_keys=key1[i]; j=i; + } } sprintf(key,"%6d",j); - if (verbose) printf(" Using key: \"%s\" Keys: %d\n",key,dupp_keys); - if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) goto err; - if (mi_rsame(file,read_record2,-1)) goto err; - if (memcmp(read_record,read_record2,reclength) != 0) - { - printf("mi_rsame didn't find same record\n"); - goto end; - } - info.recpos=mi_position(file); - if (mi_rfirst(file,read_record2,0) || - mi_rsame_with_pos(file,read_record2,0,info.recpos) || - memcmp(read_record,read_record2,reclength) != 0) + start=keyinfo[0].seg[0].start; + length=keyinfo[0].seg[0].length; + if (dupp_keys) { - printf("mi_rsame_with_pos didn't find same record\n"); - goto end; - } - { - int skr=mi_rnext(file,read_record2,0); - if ((skr && my_errno != HA_ERR_END_OF_FILE) || - mi_rprev(file,read_record2,-1) || + if (!silent) + printf("- Same key: first - next -> last - prev -> first\n"); + DBUG_PRINT("progpos",("first - next -> last - prev -> first")); + if (verbose) printf(" Using key: \"%s\" Keys: %d\n",key,dupp_keys); + + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + goto err; + if (mi_rsame(file,read_record2,-1)) + goto err; + if (memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame didn't find same record\n"); + goto end; + } + info.recpos=mi_position(file); + if (mi_rfirst(file,read_record2,0) || + mi_rsame_with_pos(file,read_record2,0,info.recpos) || memcmp(read_record,read_record2,reclength) != 0) { - printf("mi_rsame_with_pos lost position\n"); + printf("mi_rsame_with_pos didn't find same record\n"); + goto end; + } + { + int skr=mi_rnext(file,read_record2,0); + if ((skr && my_errno != HA_ERR_END_OF_FILE) || + mi_rprev(file,read_record2,-1) || + memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame_with_pos lost position\n"); + goto end; + } + } + ant=1; + while (mi_rnext(file,read_record2,0) == 0 && + memcmp(read_record2+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("next: Found: %d keys of %d\n",ant,dupp_keys); + goto end; + } + ant=0; + while (mi_rprev(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("prev: Found: %d records of %d\n",ant,dupp_keys); goto end; } - } - ant=1; - start=keyinfo[0].seg[0].start; length=keyinfo[0].seg[0].length; - while (mi_rnext(file,read_record2,0) == 0 && - memcmp(read_record2+start,key,length) == 0) ant++; - if (ant != dupp_keys) - { - printf("next: Found: %d keys of %d\n",ant,dupp_keys); - goto end; - } - ant=0; - while (mi_rprev(file,read_record3,0) == 0 && - bcmp(read_record3+start,key,length) == 0) ant++; - if (ant != dupp_keys) - { - printf("prev: Found: %d records of %d\n",ant,dupp_keys); - goto end; - } - /* Check of mi_rnext_same */ - if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) - goto err; - ant=1; - while (!mi_rnext_same(file,read_record3) && ant < dupp_keys+10) - ant++; - if (ant != dupp_keys || my_errno != HA_ERR_END_OF_FILE) - { - printf("mi_rnext_same: Found: %d records of %d\n",ant,dupp_keys); - goto end; + /* Check of mi_rnext_same */ + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + goto err; + ant=1; + while (!mi_rnext_same(file,read_record3) && ant < dupp_keys+10) + ant++; + if (ant != dupp_keys || my_errno != HA_ERR_END_OF_FILE) + { + printf("mi_rnext_same: Found: %d records of %d\n",ant,dupp_keys); + goto end; + } } if (!silent) @@ -666,7 +681,7 @@ int main(int argc, char *argv[]) if (!silent) printf("- mi_extra(CACHE) + mi_rrnd.... + mi_extra(NO_CACHE)\n"); - if (mi_extra(file,HA_EXTRA_RESET) || mi_extra(file,HA_EXTRA_CACHE)) + if (mi_extra(file,HA_EXTRA_RESET,0) || mi_extra(file,HA_EXTRA_CACHE,0)) { if (locking || (!use_blob && !pack_fields)) { @@ -684,7 +699,7 @@ int main(int argc, char *argv[]) ant,write_count-opt_delete); goto end; } - if (mi_extra(file,HA_EXTRA_NO_CACHE)) + if (mi_extra(file,HA_EXTRA_NO_CACHE,0)) { puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); goto end; @@ -709,7 +724,7 @@ int main(int argc, char *argv[]) DBUG_PRINT("progpos",("Removing keys")); lastpos = HA_OFFSET_ERROR; /* DBUG_POP(); */ - mi_extra(file,HA_EXTRA_RESET); + mi_extra(file,HA_EXTRA_RESET,0); found_parts=0; while ((error=mi_rrnd(file,read_record,HA_OFFSET_ERROR)) != HA_ERR_END_OF_FILE) @@ -776,9 +791,15 @@ end: printf("\nFollowing test have been made:\n"); printf("Write records: %d\nUpdate records: %d\nSame-key-read: %d\nDelete records: %d\n", write_count,update,dupp_keys,opt_delete); if (rec_pointer_size) - printf("Record pointer size: %d\n",rec_pointer_size); + printf("Record pointer size: %d\n",rec_pointer_size); + printf("myisam_block_size: %u\n", myisam_block_size); if (key_cacheing) - puts("Key cacheing used"); + { + puts("Key cache used"); + printf("key_cache_block_size: %u\n", key_cache_block_size); + if (write_cacheing) + puts("Key cache resized"); + } if (write_cacheing) puts("Write cacheing used"); if (write_cacheing) @@ -801,7 +822,7 @@ reads: %10lu\n", end_key_cache(); if (blob_buffer) my_free(blob_buffer,MYF(0)); - my_end(MY_CHECK_ERROR | MY_GIVE_INFO); + my_end(silent ? MY_CHECK_ERROR : MY_CHECK_ERROR | MY_GIVE_INFO); return(0); err: printf("got error: %d when using MyISAM-database\n",my_errno); @@ -861,7 +882,29 @@ static void get_options(int argc, char **argv) verbose=1; break; case 'm': /* records */ - recant=atoi(++pos); + if ((recant=atoi(++pos)) < 10) + { + fprintf(stderr,"record count must be >= 10\n"); + exit(1); + } + break; + case 'e': /* myisam_block_length */ + if ((myisam_block_size=atoi(++pos)) < MI_MIN_KEY_BLOCK_LENGTH || + myisam_block_size > MI_MAX_KEY_BLOCK_LENGTH) + { + fprintf(stderr,"Wrong myisam_block_length\n"); + exit(1); + } + myisam_block_size=1 << my_bit_log2(myisam_block_size); + break; + case 'E': /* myisam_block_length */ + if ((key_cache_block_size=atoi(++pos)) < MI_MIN_KEY_BLOCK_LENGTH || + key_cache_block_size > MI_MAX_KEY_BLOCK_LENGTH) + { + fprintf(stderr,"Wrong key_cache_block_size\n"); + exit(1); + } + key_cache_block_size=1 << my_bit_log2(key_cache_block_size); break; case 'f': if ((first_key=atoi(++pos)) < 0 || first_key >= MYISAM_KEYS) @@ -904,7 +947,7 @@ static void get_options(int argc, char **argv) case 'V': printf("%s Ver 1.2 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE); puts("By Monty, for your professional use\n"); - printf("Usage: %s [-?AbBcDIKLPRqSsVWltv] [-k#] [-f#] [-m#] [-t#]\n", + printf("Usage: %s [-?AbBcDIKLPRqSsVWltv] [-k#] [-f#] [-m#] [-e#] [-E#] [-t#]\n", progname); exit(0); case '#': diff --git a/myisam/mi_test3.c b/myisam/mi_test3.c index 48325b2dcac..6111167b38f 100644 --- a/myisam/mi_test3.c +++ b/myisam/mi_test3.c @@ -1,21 +1,23 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* Test av locking */ +#ifndef __NETWARE__ + #include "myisam.h" #include <sys/types.h> #ifdef HAVE_SYS_WAIT_H @@ -173,7 +175,7 @@ void start_test(int id) exit(1); } if (key_cacheing && rnd(2) == 0) - init_key_cache(65536L,(uint) IO_SIZE*4*10); + init_key_cache(65536L); printf("Process %d, pid: %d\n",id,getpid()); fflush(stdout); for (error=i=0 ; i < tests && !error; i++) @@ -302,7 +304,7 @@ int test_rrnd(MI_INFO *file,int id) return 1; } if (rnd(2) == 0) - mi_extra(file,HA_EXTRA_CACHE); + mi_extra(file,HA_EXTRA_CACHE,0); } count=0; @@ -323,7 +325,7 @@ int test_rrnd(MI_INFO *file,int id) end: if (lock) { - mi_extra(file,HA_EXTRA_NO_CACHE); + mi_extra(file,HA_EXTRA_NO_CACHE,0); if (mi_lock_database(file,F_UNLCK)) { fprintf(stderr,"%2d: Can't unlock table\n",id); @@ -355,7 +357,7 @@ int test_write(MI_INFO *file,int id,int lock_type) return 1; } if (rnd(2) == 0) - mi_extra(file,HA_EXTRA_WRITE_CACHE); + mi_extra(file,HA_EXTRA_WRITE_CACHE,0); } sprintf(record.id,"%7d",getpid()); @@ -380,7 +382,7 @@ int test_write(MI_INFO *file,int id,int lock_type) } if (lock) { - mi_extra(file,HA_EXTRA_NO_CACHE); + mi_extra(file,HA_EXTRA_NO_CACHE,0); if (mi_lock_database(file,F_UNLCK)) { fprintf(stderr,"%2d: Can't unlock table\n",id); @@ -483,3 +485,15 @@ int test_update(MI_INFO *file,int id,int lock_type) printf("%2d: update: %5d\n",id,update); fflush(stdout); return 0; } + +#else /* __NETWARE__ */ + +#include <stdio.h> + +main() +{ + fprintf(stderr,"this test has not been ported to NetWare\n"); + return 0; +} + +#endif /* __NETWARE__ */ diff --git a/myisam/mi_test_all.res b/myisam/mi_test_all.res index d6b4703d702..94355bf1aa2 100644 --- a/myisam/mi_test_all.res +++ b/myisam/mi_test_all.res @@ -1,48 +1,50 @@ -Maximum memory usage: 545477 bytes (533k) -Maximum memory usage: 545477 bytes (533k) -Maximum memory usage: 545369 bytes (533k) -mi_test2-i686 -s -L -K -R1 -m2000 ; Should give error 135 +mi_test2 -s -L -K -R1 -m2000 ; Should give error 135 Error: 135 in write at record: 1105 got error: 135 when using MyISAM-database -Maximum memory usage: 29439 bytes (29k) -Maximum memory usage: 66541 bytes (65k) -Maximum memory usage: 5101 bytes (5k) -Maximum memory usage: 545488 bytes (533k) +myisamchk: MyISAM file test2 +myisamchk: warning: Datafile is almost full, 65532 of 65534 used +MyISAM-table 'test2' is usable but should be fixed Commands Used count Errors Recover errors -open 1 0 0 -write 50 0 0 -update 5 0 0 -delete 50 0 0 -close 1 0 0 -extra 6 0 0 -Total 113 0 0 -Maximum memory usage: 545744 bytes (533k) +open 17 0 0 +write 850 0 0 +update 85 0 0 +delete 850 0 0 +close 17 0 0 +extra 102 0 0 +Total 1921 0 0 Commands Used count Errors Recover errors -open 2 0 0 -write 100 0 0 -update 10 0 0 -delete 100 0 0 -close 2 0 0 -extra 12 0 0 -Total 226 0 0 -Maximum memory usage: 5101 bytes (5k) -1.12user 0.52system 0:01.71elapsed 95%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (214major+25minor)pagefaults 0swaps -Maximum memory usage: 21189 bytes (21k) -1.29user 0.55system 0:01.82elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (215major+29minor)pagefaults 0swaps -Maximum memory usage: 66541 bytes (65k) -1.10user 0.51system 0:01.66elapsed 96%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (216major+40minor)pagefaults 0swaps -Maximum memory usage: 82629 bytes (81k) -1.33user 0.24system 0:01.54elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (216major+44minor)pagefaults 0swaps -Maximum memory usage: 545477 bytes (533k) -1.31user 0.24system 0:01.55elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (216major+173minor)pagefaults 0swaps -Maximum memory usage: 545389 bytes (533k) -1.37user 0.18system 0:01.57elapsed 98%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (218major+180minor)pagefaults 0swaps -Maximum memory usage: 109165 bytes (107k) -1.42user 0.38system 0:01.78elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k -0inputs+0outputs (218major+51minor)pagefaults 0swaps +open 18 0 0 +write 900 0 0 +update 90 0 0 +delete 900 0 0 +close 18 0 0 +extra 108 0 0 +Total 2034 0 0 + +real 0m1.054s +user 0m0.410s +sys 0m0.640s + +real 0m1.077s +user 0m0.550s +sys 0m0.530s + +real 0m1.100s +user 0m0.420s +sys 0m0.680s + +real 0m0.783s +user 0m0.590s +sys 0m0.200s + +real 0m0.764s +user 0m0.560s +sys 0m0.210s + +real 0m0.699s +user 0m0.570s +sys 0m0.130s + +real 0m0.991s +user 0m0.630s +sys 0m0.350s diff --git a/myisam/mi_test_all.sh b/myisam/mi_test_all.sh index ccc9c39c64e..a2d57ea1a83 100755 --- a/myisam/mi_test_all.sh +++ b/myisam/mi_test_all.sh @@ -66,6 +66,14 @@ myisamchk$suffix -rs test1 myisamchk$suffix -se test1 myisamchk$suffix -rqs test1 myisamchk$suffix -se test1 +myisamchk$suffix -rs --correct-checksum test1 +myisamchk$suffix -se test1 +myisamchk$suffix -rqs --correct-checksum test1 +myisamchk$suffix -se test1 +myisamchk$suffix -ros --correct-checksum test1 +myisamchk$suffix -se test1 +myisamchk$suffix -rqos --correct-checksum test1 +myisamchk$suffix -se test1 # check of myisampack / myisamchk myisampack$suffix --force -s test1 @@ -105,13 +113,25 @@ mi_test1$suffix $silent --key_multiple -P -S myisamchk$suffix -sm test1 mi_test2$suffix $silent -L -K -W -P +myisamchk$suffix -sm test2 mi_test2$suffix $silent -L -K -W -P -A +myisamchk$suffix -sm test2 mi_test2$suffix $silent -L -K -W -P -S -R1 -m500 echo "mi_test2$suffix $silent -L -K -R1 -m2000 ; Should give error 135" +myisamchk$suffix -sm test2 mi_test2$suffix $silent -L -K -R1 -m2000 +myisamchk$suffix -sm test2 mi_test2$suffix $silent -L -K -P -S -R3 -m50 -b1000000 +myisamchk$suffix -sm test2 mi_test2$suffix $silent -L -B +myisamchk$suffix -sm test2 mi_test2$suffix $silent -D -B -c +myisamchk$suffix -sm test2 +mi_test2$suffix $silent -m10000 -e8192 -K +myisamchk$suffix -sm test2 +mi_test2$suffix $silent -m10000 -e16384 -E16384 -K -L +myisamchk$suffix -sm test2 + mi_test2$suffix $silent -L -K -W -P -m50 -l myisamlog$suffix mi_test2$suffix $silent -L -K -W -P -m50 -l -b100 diff --git a/myisam/mi_unique.c b/myisam/mi_unique.c index 7f1e6b83a12..ddba40214e7 100644 --- a/myisam/mi_unique.c +++ b/myisam/mi_unique.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -179,19 +179,19 @@ int mi_unique_comp(MI_UNIQUEDEF *def, const byte *a, const byte *b, memcpy_fixed((byte*) &pos_a,pos_a+keyseg->bit_start,sizeof(char*)); memcpy_fixed((byte*) &pos_b,pos_b+keyseg->bit_start,sizeof(char*)); } - end= pos_a+length; if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT) { - uchar *sort_order=keyseg->charset->sort_order; - while (pos_a != end) - if (sort_order[*(uchar*) pos_a++] != - sort_order[*(uchar*) pos_b++]) + if (_mi_compare_text(keyseg->charset, (uchar *)pos_a, length, + (uchar *)pos_b, length, 0)) return 1; } else + { + end= pos_a+length; while (pos_a != end) if (*pos_a++ != *pos_b++) return 1; + } } return 0; } diff --git a/myisam/mi_update.c b/myisam/mi_update.c index 185624f3878..1614d6ca19d 100644 --- a/myisam/mi_update.c +++ b/myisam/mi_update.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -83,8 +83,6 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec) /* Check which keys changed from the original row */ new_key=info->lastkey2; - key_changed=HA_STATE_KEY_CHANGED; /* We changed current database */ - /* Remove key that didn't change */ changed=0; for (i=0 ; i < share->base.keys ; i++) { @@ -93,14 +91,12 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec) /* The following code block is for text searching by SerG */ if (share->keyinfo[i].flag & HA_FULLTEXT ) { - if(_mi_ft_cmp(info,i,oldrec, newrec)) + if (_mi_ft_cmp(info,i,oldrec, newrec)) { if ((int) i == info->lastinx) key_changed|=HA_STATE_WRITTEN; changed|=((ulonglong) 1 << i); - if (_mi_ft_del(info,i,(char*) old_key,oldrec,pos)) - goto err; - if (_mi_ft_add(info,i,(char*) new_key,newrec,pos)) + if (_mi_ft_update(info,i,(char*) old_key,oldrec,newrec,pos)) goto err; } } @@ -123,11 +119,28 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec) } } } + /* + If we are running with external locking, we must update the index file + that something has changed + */ + if (changed || !my_disable_locking) + key_changed|= HA_STATE_KEY_CHANGED; if (share->calc_checksum) + { info->checksum=(*share->calc_checksum)(info,newrec); - if ((*share->update_record)(info,pos,newrec)) - goto err; + key_changed|= HA_STATE_KEY_CHANGED; /* Must update index file */ + } + { + /* Don't update index file if data file is not extended */ + MI_STATUS_INFO state; + memcpy((char*) &state, (char*) info->state, sizeof(state)); + if ((*share->update_record)(info,pos,newrec)) + goto err; + if (!key_changed && + memcmp((char*) &state, (char*) info->state, sizeof(state))) + key_changed|= HA_STATE_KEY_CHANGED; /* Must update index file */ + } if (auto_key_changed) update_auto_increment(info,newrec); if (share->calc_checksum) @@ -138,11 +151,19 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec) myisam_log_record(MI_LOG_UPDATE,info,newrec,info->lastpos,0); VOID(_mi_writeinfo(info,key_changed ? WRITEINFO_UPDATE_KEYFILE : 0)); allow_break(); /* Allow SIGHUP & SIGINT */ + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (update)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } DBUG_RETURN(0); err: DBUG_PRINT("error",("key: %d errno: %d",i,my_errno)); save_errno=my_errno; + if (changed) + key_changed|= HA_STATE_KEY_CHANGED; if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL) { info->errkey= (int) i; diff --git a/myisam/mi_write.c b/myisam/mi_write.c index 7b468395166..d39bbbf5fc7 100644 --- a/myisam/mi_write.c +++ b/myisam/mi_write.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -25,7 +25,8 @@ /* Functions declared in this file */ -static int w_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, +static int w_search(MI_INFO *info,MI_KEYDEF *keyinfo, + uint comp_flag, uchar *key, uint key_length, my_off_t pos, uchar *father_buff, uchar *father_keypos, my_off_t father_page, my_bool insert_last); @@ -35,17 +36,21 @@ static int _mi_balance_page(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page, uchar *key, uint *return_key_length, uchar **after_key); - +int _mi_ck_write_tree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length); +int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length); /* Write new record to database */ int mi_write(MI_INFO *info, byte *record) { + MYISAM_SHARE *share=info->s; uint i; int save_errno; my_off_t filepos; uchar *buff; - MYISAM_SHARE *share=info->s; + my_bool lock_tree= share->concurrent_insert; DBUG_ENTER("mi_write"); DBUG_PRINT("enter",("isam: %d data: %d",info->s->kfile,info->dfile)); @@ -96,33 +101,36 @@ int mi_write(MI_INFO *info, byte *record) { if (((ulonglong) 1 << i) & share->state.key_map) { - if (share->concurrent_insert) + bool local_lock_tree= (lock_tree && + !(info->bulk_insert && + is_tree_inited(&info->bulk_insert[i]))); + if (local_lock_tree) { rw_wrlock(&share->key_root_lock[i]); share->keyinfo[i].version++; } - if (share->keyinfo[i].flag & HA_FULLTEXT ) /* SerG */ - { /* SerG */ - if (_mi_ft_add(info,i,(char*) buff,record,filepos)) /* SerG */ - { /* SerG */ - if (share->concurrent_insert) + if (share->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_add(info,i,(char*) buff,record,filepos)) + { + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); - DBUG_PRINT("error",("Got error: %d on write",my_errno)); /* SerG */ - goto err; /* SerG */ - } /* SerG */ - } /* SerG */ - else /* SerG */ + DBUG_PRINT("error",("Got error: %d on write",my_errno)); + goto err; + } + } + else { uint key_length=_mi_make_key(info,i,buff,record,filepos); if (_mi_ck_write(info,i,buff,key_length)) { - if (share->concurrent_insert) + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); DBUG_PRINT("error",("Got error: %d on write",my_errno)); goto err; } } - if (share->concurrent_insert) + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); } } @@ -142,6 +150,12 @@ int mi_write(MI_INFO *info, byte *record) info->lastpos=filepos; myisam_log_record(MI_LOG_WRITE,info,record,filepos,0); VOID(_mi_writeinfo(info, WRITEINFO_UPDATE_KEYFILE)); + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (update)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } allow_break(); /* Allow SIGHUP & SIGINT */ DBUG_RETURN(0); @@ -149,19 +163,32 @@ err: save_errno=my_errno; if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL) { + if (info->bulk_insert) + { + uint j; + for (j=0 ; j < share->base.keys ; j++) + { + if (is_tree_inited(&info->bulk_insert[j])) + { + reset_tree(&info->bulk_insert[j]); + } + } + } info->errkey= (int) i; while ( i-- > 0) { if (((ulonglong) 1 << i) & share->state.key_map) { - if (share->concurrent_insert) + bool local_lock_tree= (lock_tree && + !(info->bulk_insert && + is_tree_inited(&info->bulk_insert[i]))); + if (local_lock_tree) rw_wrlock(&share->key_root_lock[i]); - /* The following code block is for text searching by SerG */ if (share->keyinfo[i].flag & HA_FULLTEXT) { if (_mi_ft_del(info,i,(char*) buff,record,filepos)) { - if (share->concurrent_insert) + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); break; } @@ -171,12 +198,12 @@ err: uint key_length=_mi_make_key(info,i,buff,record,filepos); if (_mi_ck_delete(info,i,buff,key_length)) { - if (share->concurrent_insert) + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); break; } } - if (share->concurrent_insert) + if (local_lock_tree) rw_unlock(&share->key_root_lock[i]); } } @@ -196,19 +223,51 @@ err2: /* Write one key to btree */ -int _mi_ck_write(register MI_INFO *info, uint keynr, uchar *key, - uint key_length) +int _mi_ck_write(MI_INFO *info, uint keynr, uchar *key, uint key_length) { - int error; DBUG_ENTER("_mi_ck_write"); + if (info->bulk_insert && is_tree_inited(&info->bulk_insert[keynr])) + { + DBUG_RETURN(_mi_ck_write_tree(info, keynr, key, key_length)); + } + else + { + DBUG_RETURN(_mi_ck_write_btree(info, keynr, key, key_length)); + } +} /* _mi_ck_write */ + + +/********************************************************************** + * Normal insert code * + **********************************************************************/ + +int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + uint comp_flag; + MI_KEYDEF *keyinfo=info->s->keyinfo+keynr; + DBUG_ENTER("_mi_ck_write_btree"); + + if (keyinfo->flag & HA_SORT_ALLOWS_SAME) + comp_flag=SEARCH_BIGGER; /* Put after same key */ + else if (keyinfo->flag & HA_NOSAME) + { + comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* No dupplicates */ + if (keyinfo->flag & HA_NULL_ARE_EQUAL) + comp_flag|= SEARCH_NULL_ARE_EQUAL; + } + else + comp_flag=SEARCH_SAME; /* Keys in rec-pos order */ + if (info->s->state.key_root[keynr] == HA_OFFSET_ERROR || - (error=w_search(info,info->s->keyinfo+keynr,key, key_length, + (error=w_search(info, keyinfo, comp_flag, key, key_length, info->s->state.key_root[keynr], (uchar *) 0, (uchar*) 0, (my_off_t) 0, 1)) > 0) error=_mi_enlarge_root(info,keynr,key); DBUG_RETURN(error); -} /* _mi_ck_write */ +} /* _mi_ck_write_btree */ /* Make a new root with key as only pointer */ @@ -246,13 +305,12 @@ int _mi_enlarge_root(register MI_INFO *info, uint keynr, uchar *key) */ static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, - uchar *key, uint key_length, my_off_t page, - uchar *father_buff, - uchar *father_keypos, my_off_t father_page, - my_bool insert_last) + uint comp_flag, uchar *key, uint key_length, my_off_t page, + uchar *father_buff, uchar *father_keypos, + my_off_t father_page, my_bool insert_last) { int error,flag; - uint comp_flag,nod_flag; + uint nod_flag, search_key_length; uchar *temp_buff,*keypos; uchar keybuff[MI_MAX_KEY_BUFF]; my_bool was_last_key; @@ -260,25 +318,15 @@ static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, DBUG_ENTER("w_search"); DBUG_PRINT("enter",("page: %ld",page)); - if (keyinfo->flag & HA_SORT_ALLOWS_SAME) - comp_flag=SEARCH_BIGGER; /* Put after same key */ - else if (keyinfo->flag & HA_NOSAME) - { - comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* No dupplicates */ - if (keyinfo->flag & HA_NULL_ARE_EQUAL) - comp_flag|= SEARCH_NULL_ARE_EQUAL; - } - else - comp_flag=SEARCH_SAME; /* Keys in rec-pos order */ - + search_key_length= (comp_flag & SEARCH_FIND) ? key_length : USE_WHOLE_KEY; if (!(temp_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ MI_MAX_KEY_BUFF*2))) DBUG_RETURN(-1); if (!_mi_fetch_keypage(info,keyinfo,page,temp_buff,0)) goto err; - flag=(*keyinfo->bin_search)(info,keyinfo,temp_buff,key,key_length,comp_flag, - &keypos, keybuff, &was_last_key); + flag=(*keyinfo->bin_search)(info,keyinfo,temp_buff,key,search_key_length, + comp_flag, &keypos, keybuff, &was_last_key); nod_flag=mi_test_if_nod(temp_buff); if (flag == 0) { @@ -299,7 +347,7 @@ static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, insert_last=0; next_page=_mi_kpos(nod_flag,keypos); if (next_page == HA_OFFSET_ERROR || - (error=w_search(info,keyinfo,key,key_length,next_page, + (error=w_search(info, keyinfo, comp_flag, key, key_length, next_page, temp_buff, keypos, page, insert_last)) >0) { error=_mi_insert(info,keyinfo,key,temp_buff,keypos,keybuff,father_buff, @@ -349,7 +397,7 @@ int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo, { DBUG_PRINT("test",("t_length: %d ref_len: %d", t_length,s_temp.ref_length)); - DBUG_PRINT("test",("n_ref_len: %d n_length: %d key: %lx", + DBUG_PRINT("test",("n_ref_len: %d n_length: %d key_pos: %lx", s_temp.n_ref_length,s_temp.n_length,s_temp.key)); } #endif @@ -686,3 +734,157 @@ static int _mi_balance_page(register MI_INFO *info, MI_KEYDEF *keyinfo, err: DBUG_RETURN(-1); } /* _mi_balance_page */ + +/********************************************************************** + * Bulk insert code * + **********************************************************************/ + +typedef struct { + MI_INFO *info; + uint keynr; +} bulk_insert_param; + +int _mi_ck_write_tree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + DBUG_ENTER("_mi_ck_write_tree"); + + error= tree_insert(&info->bulk_insert[keynr], key, + key_length + info->s->rec_reflength) ? 0 : HA_ERR_OUT_OF_MEM ; + + DBUG_RETURN(error); +} /* _mi_ck_write_tree */ + + +/* typeof(_mi_keys_compare)=qsort_cmp2 */ + +static int keys_compare(bulk_insert_param *param, uchar *key1, uchar *key2) +{ + uint not_used; + return _mi_key_cmp(param->info->s->keyinfo[param->keynr].seg, + key1, key2, USE_WHOLE_KEY, SEARCH_SAME, + ¬_used); +} + + +static int keys_free(uchar *key, TREE_FREE mode, bulk_insert_param *param) +{ + /* + Probably I can use info->lastkey here, but I'm not sure, + and to be safe I'd better use local lastkey. + */ + uchar lastkey[MI_MAX_KEY_BUFF]; + uint keylen; + MI_KEYDEF *keyinfo; + + switch (mode) { + case free_init: + if (param->info->s->concurrent_insert) + { + rw_wrlock(¶m->info->s->key_root_lock[param->keynr]); + param->info->s->keyinfo[param->keynr].version++; + } + return 0; + case free_free: + keyinfo=param->info->s->keyinfo+param->keynr; + keylen=_mi_keylength(keyinfo, key); + memcpy(lastkey, key, keylen); + return _mi_ck_write_btree(param->info,param->keynr,lastkey, + keylen - param->info->s->rec_reflength); + case free_end: + if (param->info->s->concurrent_insert) + rw_unlock(¶m->info->s->key_root_lock[param->keynr]); + return 0; + } + return -1; +} + + +int mi_init_bulk_insert(MI_INFO *info, ulong cache_size, ha_rows rows) +{ + MYISAM_SHARE *share=info->s; + MI_KEYDEF *key=share->keyinfo; + bulk_insert_param *params; + uint i, num_keys, total_keylength; + ulonglong key_map=0; + DBUG_ENTER("_mi_init_bulk_insert"); + DBUG_PRINT("enter",("cache_size: %lu", cache_size)); + + if (info->bulk_insert || (rows && rows < MI_MIN_ROWS_TO_USE_BULK_INSERT)) + DBUG_RETURN(0); + + for (i=total_keylength=num_keys=0 ; i < share->base.keys ; i++) + { + if (!(key[i].flag & HA_NOSAME) && share->base.auto_key != i+1 + && test(share->state.key_map & ((ulonglong) 1 << i))) + { + num_keys++; + key_map |=((ulonglong) 1 << i); + total_keylength+=key[i].maxlength+TREE_ELEMENT_EXTRA_SIZE; + } + } + + if (num_keys==0 || + num_keys * MI_MIN_SIZE_BULK_INSERT_TREE > cache_size) + DBUG_RETURN(0); + + if (rows && rows*total_keylength < cache_size) + cache_size=rows; + else + cache_size/=total_keylength*16; + + info->bulk_insert=(TREE *) + my_malloc((sizeof(TREE)*share->base.keys+ + sizeof(bulk_insert_param)*num_keys),MYF(0)); + + if (!info->bulk_insert) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + + params=(bulk_insert_param *)(info->bulk_insert+share->base.keys); + for (i=0 ; i < share->base.keys ; i++) + { + if (test(key_map & ((ulonglong) 1 << i))) + { + params->info=info; + params->keynr=i; + /* Only allocate a 16'th of the buffer at a time */ + init_tree(&info->bulk_insert[i], + cache_size * key[i].maxlength, + cache_size * key[i].maxlength, 0, + (qsort_cmp2)keys_compare, 0, + (tree_element_free) keys_free, (void *)params++); + } + else + info->bulk_insert[i].root=0; + } + + DBUG_RETURN(0); +} + +void mi_flush_bulk_insert(MI_INFO *info, uint inx) +{ + if (info->bulk_insert) + { + if (is_tree_inited(&info->bulk_insert[inx])) + reset_tree(&info->bulk_insert[inx]); + } +} + +void mi_end_bulk_insert(MI_INFO *info) +{ + if (info->bulk_insert) + { + uint i; + for (i=0 ; i < info->s->base.keys ; i++) + { + if (is_tree_inited(& info->bulk_insert[i])) + { + delete_tree(& info->bulk_insert[i]); + } + } + my_free((void *)info->bulk_insert, MYF(0)); + info->bulk_insert=0; + } +} + diff --git a/myisam/myisamchk.c b/myisam/myisamchk.c index cb321f8fda3..ac1d0fbfc4a 100644 --- a/myisam/myisamchk.c +++ b/myisam/myisamchk.c @@ -1,27 +1,26 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Descript, check and repair of ISAM tables */ +/* Describe, check and repair of MyISAM tables */ #include "fulltext.h" #include <m_ctype.h> #include <stdarg.h> -#include <getopt.h> -#include <assert.h> +#include <my_getopt.h> #ifdef HAVE_SYS_VADVICE_H #include <sys/vadvise.h> #endif @@ -44,6 +43,7 @@ static char **default_argv; static const char *load_default_groups[]= { "myisamchk", 0 }; static const char *set_charset_name; static CHARSET_INFO *set_charset; +static long opt_myisam_block_size; static const char *type_names[]= { "?","char","binary", "short", "long", "float", @@ -69,12 +69,11 @@ static void print_version(void); static void usage(void); static int myisamchk(MI_CHECK *param, char *filename); static void descript(MI_CHECK *param, register MI_INFO *info, my_string name); -static int mi_sort_records(MI_CHECK *param, - register MI_INFO *info, my_string name, - uint sort_key, - my_bool write_info, - my_bool update_index); -static int sort_record_index(MI_CHECK *param,MI_INFO *info,MI_KEYDEF *keyinfo, +static int mi_sort_records(MI_CHECK *param, register MI_INFO *info, + my_string name, uint sort_key, + my_bool write_info, my_bool update_index); +static int sort_record_index(MI_SORT_PARAM *sort_param, MI_INFO *info, + MI_KEYDEF *keyinfo, my_off_t page,uchar *buff,uint sortkey, File new_file, my_bool update_index); @@ -100,12 +99,13 @@ int main(int argc, char **argv) while (--argc >= 0) { int new_error=myisamchk(&check_param, *(argv++)); + if ((check_param.testflag & T_REP_ANY) != T_REP) + check_param.testflag&= ~T_REP; VOID(fflush(stdout)); VOID(fflush(stderr)); if ((check_param.error_printed | check_param.warning_printed) && (check_param.testflag & T_FORCE_CREATE) && - (!(check_param.testflag & (T_REP | T_REP_BY_SORT | T_SORT_RECORDS | - T_SORT_INDEX)))) + (!(check_param.testflag & (T_REP | T_SORT_RECORDS | T_SORT_INDEX)))) { uint old_testflag=check_param.testflag; if (!(check_param.testflag & T_REP)) @@ -129,7 +129,7 @@ int main(int argc, char **argv) char buff[22],buff2[22]; if (!(check_param.testflag & T_SILENT) || check_param.testflag & T_INFO) puts("\n---------\n"); - printf("\nTotal of all %d ISAM-files:\nData records: %9s Deleted blocks: %9s\n",check_param.total_files,llstr(check_param.total_records,buff), + printf("\nTotal of all %d MyISAM-files:\nData records: %9s Deleted blocks: %9s\n",check_param.total_files,llstr(check_param.total_records,buff), llstr(check_param.total_deleted,buff2)); } free_defaults(default_argv); @@ -141,112 +141,233 @@ int main(int argc, char **argv) #endif } /* main */ +enum options { + OPT_CHARSETS_DIR=256, OPT_SET_CHARSET,OPT_START_CHECK_POS, + OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE, OPT_MYISAM_BLOCK_SIZE, + OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, + OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, + OPT_FT_MAX_WORD_LEN, OPT_FT_MAX_WORD_LEN_FOR_SORT +}; -static CHANGEABLE_VAR changeable_vars[] = { - { "key_buffer_size",(long*) &check_param.use_buffers,(long) USE_BUFFER_INIT, - (long) MALLOC_OVERHEAD, (long) ~0L,(long) MALLOC_OVERHEAD,(long) IO_SIZE }, - { "read_buffer_size", (long*) &check_param.read_buffer_length,(long) READ_BUFFER_INIT, - (long) MALLOC_OVERHEAD,(long) ~0L,(long) MALLOC_OVERHEAD,(long) 1L }, - { "write_buffer_size", (long*) &check_param.write_buffer_length,(long) READ_BUFFER_INIT, - (long) MALLOC_OVERHEAD,(long) ~0L,(long) MALLOC_OVERHEAD,(long) 1L }, - { "sort_buffer_size",(long*) &check_param.sort_buffer_length,(long) SORT_BUFFER_INIT, - (long) (MIN_SORT_BUFFER+MALLOC_OVERHEAD),(long) ~0L, - (long) MALLOC_OVERHEAD,(long) 1L }, - { "sort_key_blocks",(long*) &check_param.sort_key_blocks,BUFFERS_WHEN_SORTING,4L,100L,0L, - 1L }, - { "decode_bits",(long*) &decode_bits,9L,4L,17L,0L,1L }, - { NullS,(long*) 0,0L,0L,0L,0L,0L,} }; - -enum options {OPT_CHARSETS_DIR=256, OPT_SET_CHARSET,OPT_START_CHECK_POS}; - - -static struct option long_options[] = +static struct my_option my_long_options[] = { - {"analyze", no_argument, 0, 'a'}, - {"block-search", required_argument,0, 'b'}, - {"backup", no_argument, 0, 'B'}, - {"character-sets-dir",required_argument,0, OPT_CHARSETS_DIR}, - {"check", no_argument, 0, 'c'}, - {"check-only-changed",no_argument, 0, 'C'}, + {"analyze", 'a', + "Analyze distribution of keys. Will make some joins in MySQL faster. You can check the calculated distribution.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"block-search", 'b', + "No help available.", + 0, 0, 0, GET_ULONG, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"backup", 'B', + "Make a backup of the .MYD file as 'filename-time.BAK'", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"character-sets-dir", OPT_CHARSETS_DIR, + "Directory where character sets are.", + (gptr*) &set_charset_name, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"check", 'c', + "Check table for errors.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"check-only-changed", 'C', + "Check only tables that have changed since last check.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"correct-checksum", OPT_CORRECT_CHECKSUM, + "Correct checksum information for table.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF - {"debug", optional_argument, 0, '#'}, + {"debug", '#', + "Output debug log. Often this is 'd:t:o,filename'.", + 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif + {"description", 'd', + "Prints some information about table.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"data-file-length", 'D', + "Max length of data file (when recreating data-file when it's full).", + (gptr*) &check_param.max_data_file_length, + (gptr*) &check_param.max_data_file_length, + 0, GET_LL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"extend-check", 'e', + "If used when checking a table, ensure that the table is 100 percent consistent, which will take a long time. If used when repairing a table, try to recover every possible row from the data file. Normally this will also find a lot of garbage rows; Don't use this option with repair if you are not totally desperate.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"fast", 'F', + "Check only tables that haven't been closed properly.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"force", 'f', + "Restart with -r if there are any errors in the table. States will be updated as with --update-state.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"HELP", 'H', + "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"help", '?', + "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"information", 'i', + "Print statistics information about table that is checked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"keys-used", 'k', + "Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts!", + (gptr*) &check_param.keys_in_use, + (gptr*) &check_param.keys_in_use, + 0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0}, + {"medium-check", 'm', + "Faster than extend-check, but only finds 99.99% of all errors. Should be good enough for most cases.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"quick", 'q', "Faster repair by not modifying the data file.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"read-only", 'T', + "Don't mark table as checked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"recover", 'r', + "Can fix almost anything except unique keys that aren't unique.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel-recover", 'p', + "Same as '-r' but creates all the keys in parallel", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"safe-recover", 'o', + "Uses old recovery method; Slower than '-r' but can handle a couple of cases where '-r' reports that it can't fix the data file.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-recover", 'n', + "Force recovering with sorting even if the temporary file was very big.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifdef DEBUG + {"start-check-pos", OPT_START_CHECK_POS, + "No help available.", + 0, 0, 0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #endif - {"description", no_argument, 0, 'd'}, - {"data-file-length", required_argument, 0, 'D'}, - {"extend-check", no_argument, 0, 'e'}, - {"fast", no_argument, 0, 'F'}, - {"force", no_argument, 0, 'f'}, - {"help", no_argument, 0, '?'}, - {"information", no_argument, 0, 'i'}, - {"keys-used", required_argument, 0, 'k'}, - {"medium-check", no_argument, 0, 'm'}, - {"no-symlinks", no_argument, 0, 'l'}, - {"quick", no_argument, 0, 'q'}, - {"read-only", no_argument, 0, 'T'}, - {"recover", no_argument, 0, 'r'}, - {"safe-recover", no_argument, 0, 'o'}, - {"start-check-pos", required_argument, 0, OPT_START_CHECK_POS}, - {"set-auto-increment",optional_argument, 0, 'A'}, - {"set-character-set",required_argument,0,OPT_SET_CHARSET}, - {"set-variable", required_argument, 0, 'O'}, - {"silent", no_argument, 0, 's'}, - {"sort-index", no_argument, 0, 'S'}, - {"sort-records", required_argument, 0, 'R'}, - {"sort-recover", no_argument, 0, 'n'}, - {"tmpdir", required_argument, 0, 't'}, - {"update-state", no_argument, 0, 'U'}, - {"unpack", no_argument, 0, 'u'}, - {"verbose", no_argument, 0, 'v'}, - {"version", no_argument, 0, 'V'}, - {"wait", no_argument, 0, 'w'}, - {0, 0, 0, 0} + {"set-auto-increment", 'A', + "Force auto_increment to start at this or higher value. If no value is given, then sets the next auto_increment value to the highest used value for the auto key + 1.", + (gptr*) &check_param.auto_increment_value, + (gptr*) &check_param.auto_increment_value, + 0, GET_ULL, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"set-character-set", OPT_SET_CHARSET, + "Change the character set used by the index", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"set-variable", 'O', + "Change the value of a variable. Please note that this option is deprecated; you can set variables directly with --variable-name=value.", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', + "Only print errors. One can use two -s to make myisamchk very silent.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-index", 'S', + "Sort index blocks. This speeds up 'read-next' in applications.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-records", 'R', + "Sort records according to an index. This makes your data much more localized and may speed up things. (It may be VERY slow to do a sort the first time!)", + (gptr*) &check_param.opt_sort_key, + (gptr*) &check_param.opt_sort_key, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 't', + "Path for temporary files.", + (gptr*) &check_param.tmpdir, + 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"update-state", 'U', + "Mark tables as crashed if any errors were found.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"unpack", 'u', + "Unpack file packed with myisampack.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', + "Print more information. This can be used with --description and --check. Use many -v for more verbosity!", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', + "Print version and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"wait", 'w', + "Wait if table is locked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "", + (gptr*) &check_param.use_buffers, (gptr*) &check_param.use_buffers, 0, + GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, (long) MALLOC_OVERHEAD, + (long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0}, + { "myisam_block_size", OPT_MYISAM_BLOCK_SIZE, "", + (gptr*) &opt_myisam_block_size, (gptr*) &opt_myisam_block_size, 0, + GET_LONG, REQUIRED_ARG, MI_KEY_BLOCK_LENGTH, MI_MIN_KEY_BLOCK_LENGTH, + MI_MAX_KEY_BLOCK_LENGTH, 0, MI_MIN_KEY_BLOCK_LENGTH, 0}, + { "read_buffer_size", OPT_READ_BUFFER_SIZE, "", + (gptr*) &check_param.read_buffer_length, + (gptr*) &check_param.read_buffer_length, 0, GET_ULONG, REQUIRED_ARG, + (long) READ_BUFFER_INIT, (long) MALLOC_OVERHEAD, + (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0}, + { "write_buffer_size", OPT_WRITE_BUFFER_SIZE, "", + (gptr*) &check_param.write_buffer_length, + (gptr*) &check_param.write_buffer_length, 0, GET_ULONG, REQUIRED_ARG, + (long) READ_BUFFER_INIT, (long) MALLOC_OVERHEAD, + (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0}, + { "sort_buffer_size", OPT_SORT_BUFFER_SIZE, "", + (gptr*) &check_param.sort_buffer_length, + (gptr*) &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG, + (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD), + (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0}, + { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "", + (gptr*) &check_param.sort_key_blocks, + (gptr*) &check_param.sort_key_blocks, 0, GET_ULONG, REQUIRED_ARG, + BUFFERS_WHEN_SORTING, 4L, 100L, 0L, 1L, 0}, + { "decode_bits", OPT_DECODE_BITS, "", (gptr*) &decode_bits, + (gptr*) &decode_bits, 0, GET_UINT, REQUIRED_ARG, 9L, 4L, 17L, 0L, 1L, 0}, + { "ft_min_word_len", OPT_FT_MIN_WORD_LEN, "", (gptr*) &ft_min_word_len, + (gptr*) &ft_min_word_len, 0, GET_ULONG, REQUIRED_ARG, 4, 1, HA_FT_MAXLEN, + 0, 1, 0}, + { "ft_max_word_len", OPT_FT_MAX_WORD_LEN, "", (gptr*) &ft_max_word_len, + (gptr*) &ft_max_word_len, 0, GET_ULONG, REQUIRED_ARG, HA_FT_MAXLEN, 10, + HA_FT_MAXLEN, 0, 1, 0}, + { "ft_max_word_len_for_sort", OPT_FT_MAX_WORD_LEN_FOR_SORT, "", + (gptr*) &ft_max_word_len_for_sort, (gptr*) &ft_max_word_len_for_sort, 0, + GET_ULONG, REQUIRED_ARG, 20, 4, HA_FT_MAXLEN, 0, 1, 0}, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + static void print_version(void) { - printf("%s Ver 1.53 for %s at %s\n",my_progname,SYSTEM_TYPE, + printf("%s Ver 2.6 for %s at %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); } + static void usage(void) { - uint i; print_version(); puts("By Monty, for your professional use"); puts("This software comes with NO WARRANTY: see the PUBLIC for details.\n"); - puts("Description, check and repair of ISAM tables."); + puts("Description, check and repair of MyISAM tables."); puts("Used without options all tables on the command will be checked for errors"); printf("Usage: %s [OPTIONS] tables[.MYI]\n", my_progname); puts("\nGlobal options:\n\ - -#, --debug=... Output debug log. Often this is 'd:t:o,filename`\n\ + -#, --debug=... Output debug log. Often this is 'd:t:o,filename'\n\ -?, --help Display this help and exit.\n\ -O, --set-variable var=option\n\ - Change the value of a variable.\n\ + Change the value of a variable. Please note that\n\ + this option is deprecated; you can set variables\n\ + directly with '--variable-name=value'.\n\ + -t, --tmpdir=path Path for temporary files\n\ -s, --silent Only print errors. One can use two -s to make\n\ myisamchk very silent\n\ -v, --verbose Print more information. This can be used with\n\ - --describe and --check. Use many -v for more verbosity!\n\ + --description and --check. Use many -v for more verbosity!\n\ -V, --version Print version and exit.\n\ -w, --wait Wait if table is locked.\n"); +#ifdef DEBUG + puts(" --start-check-pos=# Start reading file at given offset.\n"); +#endif puts("Check options (check is the default action for myisamchk):\n\ -c, --check Check table for errors\n\ -e, --extend-check Check the table VERY throughly. Only use this in\n\ extreme cases as myisamchk should normally be able to\n\ find out if the table is ok even without this switch\n\ - -F, --fast Check only tables that hasn't been closed properly\n\ + -F, --fast Check only tables that haven't been closed properly\n\ -C, --check-only-changed\n\ - Check only tables that has changed since last check\n\ - -f, --force Restart with -r if there are any errors in the table.\n\ - States will be updated as with --update-state\n\ + Check only tables that have changed since last check\n\ + -f, --force Restart with '-r' if there are any errors in the table.\n\ + States will be updated as with '--update-state'\n\ -i, --information Print statistics information about table that is checked\n\ - -m, --medium-check Faster than extended-check, but only finds 99.99% of\n\ + -m, --medium-check Faster than extend-check, but only finds 99.99% of\n\ all errors. Should be good enough for most cases\n\ -U --update-state Mark tables as crashed if you find any errors\n\ -T, --read-only Don't mark table as checked\n"); - puts("Repair options (When using -r or -o) \n\ + puts("Repair options (When using '-r' or '-o') \n\ -B, --backup Make a backup of the .MYD file as 'filename-time.BAK'\n\ + --correct-checksum Correct checksum information for table.\n\ -D, --data-file-length=# Max length of data file (when recreating data\n\ file when it's full)\n\ -e, --extend-check Try to recover every possible row from the data file\n\ @@ -256,12 +377,14 @@ static void usage(void) -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ bit mask of which keys to use. This can be used to\n\ get faster inserts!\n\ - -l, --no-symlinks Do not follow symbolic links. Normally\n\ - myisamchk repairs the table a symlink points at.\n\ -r, --recover Can fix almost anything except unique keys that aren't\n\ unique.\n\ - -n, --sort-recover Force recovering with sorting even if the temporary\n\ + -n, --sort-recover Forces recovering with sorting even if the temporary\n\ file would be very big.\n\ + -p, --parallel-recover\n\ + Uses the same technique as '-r' and '-n', but creates\n\ + all the keys in parallel, in different threads.\n\ + THIS IS ALPHA CODE. USE AT YOUR OWN RISK!\n\ -o, --safe-recover Uses old recovery method; Slower than '-r' but can\n\ handle a couple of cases where '-r' reports that it\n\ can't fix the data file.\n\ @@ -269,7 +392,6 @@ static void usage(void) Directory where character sets are\n\ --set-character-set=name\n\ Change the character set used by the index\n\ - -t, --tmpdir=path Path for temporary files\n\ -q, --quick Faster repair by not modifying the data file.\n\ One can give a second '-q' to force myisamchk to\n\ modify the original datafile in case of duplicate keys\n\ @@ -279,7 +401,7 @@ static void usage(void) puts("Other actions:\n\ -a, --analyze Analyze distribution of keys. Will make some joins in\n\ MySQL faster. You can check the calculated distribution\n\ - by using '--describe --verbose table_name'.\n\ + by using '--description --verbose table_name'.\n\ -d, --description Prints some information about table.\n\ -A, --set-auto-increment[=value]\n\ Force auto_increment to start at this or higher value\n\ @@ -290,126 +412,183 @@ static void usage(void) -R, --sort-records=#\n\ Sort records according to an index. This makes your\n\ data much more localized and may speed up things\n\ - (It may be VERY slow to do a sort the first time!)"); - - print_defaults("my",load_default_groups); - printf("\nPossible variables for option --set-variable (-O) are:\n"); - for (i=0; changeable_vars[i].name ; i++) - printf("%-20s current value: %lu\n", - changeable_vars[i].name, - *changeable_vars[i].varptr); + (It may be VERY slow to do a sort the first time!)\n\ + -b, --block-search=#\n\ + Find a record, a block at given offset belongs to."); + + print_defaults("my", load_default_groups); + my_print_variables(my_long_options); } /* Read options */ -static void get_options(register int *argc,register char ***argv) +static my_bool +get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument) { - int c,option_index=0; - uint old_testflag; - - load_defaults("my",load_default_groups,argc,argv); - default_argv= *argv; - set_all_changeable_vars(changeable_vars); - if (isatty(fileno(stdout))) - check_param.testflag|=T_WRITE_LOOP; - while ((c=getopt_long(*argc,*argv, - "aBcCdeifF?lqrmnosSTuUvVw#:b:D:k:O:R:A::t:", - long_options, &option_index)) != EOF) - { - switch(c) { - case 'a': + switch (optid) { + case 'a': + if (argument == disabled_my_option) + check_param.testflag&= ~T_STATISTICS; + else check_param.testflag|= T_STATISTICS; - break; - case 'A': - if (optarg) - check_param.auto_increment_value=strtoull(optarg,NULL,0); - else - check_param.auto_increment_value=0; /* Set to max used value */ - check_param.testflag|= T_AUTO_INC; - break; - case 'b': - check_param.search_after_block=strtoul(optarg,NULL,10); - break; - case 'B': + break; + case 'A': + if (argument) + check_param.auto_increment_value= strtoull(argument, NULL, 0); + else + check_param.auto_increment_value= 0; /* Set to max used value */ + check_param.testflag|= T_AUTO_INC; + break; + case 'b': + check_param.search_after_block= strtoul(argument, NULL, 10); + break; + case 'B': + if (argument == disabled_my_option) + check_param.testflag&= ~T_BACKUP_DATA; + else check_param.testflag|= T_BACKUP_DATA; - break; - case 'c': + break; + case 'c': + if (argument == disabled_my_option) + check_param.testflag&= ~T_CHECK; + else check_param.testflag|= T_CHECK; - break; - case 'C': + break; + case 'C': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_CHECK | T_CHECK_ONLY_CHANGED); + else check_param.testflag|= T_CHECK | T_CHECK_ONLY_CHANGED; - break; - case 'D': - check_param.max_data_file_length=strtoll(optarg,NULL,10); - break; - case 's': /* silent */ + break; + case 'D': + check_param.max_data_file_length=strtoll(argument, NULL, 10); + break; + case 's': /* silent */ + if (argument == disabled_my_option) + check_param.testflag&= ~(T_SILENT | T_VERY_SILENT); + else + { if (check_param.testflag & T_SILENT) - check_param.testflag|=T_VERY_SILENT; + check_param.testflag|= T_VERY_SILENT; check_param.testflag|= T_SILENT; check_param.testflag&= ~T_WRITE_LOOP; - break; - case 'w': + } + break; + case 'w': + if (argument == disabled_my_option) + check_param.testflag&= ~T_WAIT_FOREVER; + else check_param.testflag|= T_WAIT_FOREVER; - break; - case 'd': /* description if isam-file */ + break; + case 'd': /* description if isam-file */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_DESCRIPT; + else check_param.testflag|= T_DESCRIPT; - break; - case 'e': /* extend check */ + break; + case 'e': /* extend check */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_EXTEND; + else check_param.testflag|= T_EXTEND; - break; - case 'i': + break; + case 'i': + if (argument == disabled_my_option) + check_param.testflag&= ~T_INFO; + else check_param.testflag|= T_INFO; - break; - case 'f': + break; + case 'f': + if (argument == disabled_my_option) + { + check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; + check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE); + } + else + { check_param.tmpfile_createflag= O_RDWR | O_TRUNC; check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; - break; - case 'F': - check_param.testflag|=T_FAST; - break; - case 'k': - check_param.keys_in_use= (ulonglong) strtoll(optarg,NULL,10); - break; - case 'l': - check_param.opt_follow_links=0; - break; - case 'm': + } + break; + case 'F': + if (argument == disabled_my_option) + check_param.testflag&= ~T_FAST; + else + check_param.testflag|= T_FAST; + break; + case 'k': + check_param.keys_in_use= (ulonglong) strtoll(argument, NULL, 10); + break; + case 'm': + if (argument == disabled_my_option) + check_param.testflag&= ~T_MEDIUM; + else check_param.testflag|= T_MEDIUM; /* Medium check */ - break; - case 'r': /* Repair table */ - check_param.testflag= (check_param.testflag & ~T_REP) | T_REP_BY_SORT; - break; - case 'o': - check_param.testflag= (check_param.testflag & ~T_REP_BY_SORT) | T_REP; - check_param.force_sort=0; - my_disable_async_io=1; /* More safety */ - break; - case 'n': - check_param.testflag= (check_param.testflag & ~T_REP) | T_REP_BY_SORT; - check_param.force_sort=1; - break; - case 'q': - check_param.opt_rep_quick++; - break; - case 'u': + break; + case 'r': /* Repair table */ + check_param.testflag&= ~T_REP_ANY; + if (argument != disabled_my_option) + check_param.testflag|= T_REP_BY_SORT; + break; + case 'p': + check_param.testflag&= ~T_REP_ANY; + if (argument != disabled_my_option) + check_param.testflag|= T_REP_PARALLEL; + break; + case 'o': + check_param.testflag&= ~T_REP_ANY; + check_param.force_sort= 0; + if (argument != disabled_my_option) + { + check_param.testflag|= T_REP; + my_disable_async_io= 1; /* More safety */ + } + break; + case 'n': + check_param.testflag&= ~T_REP_ANY; + if (argument == disabled_my_option) + check_param.force_sort= 0; + else + { + check_param.testflag|= T_REP_BY_SORT; + check_param.force_sort= 1; + } + break; + case 'q': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); + else + check_param.testflag|= + (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK; + break; + case 'u': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_UNPACK | T_REP_BY_SORT); + else check_param.testflag|= T_UNPACK | T_REP_BY_SORT; - break; - case 'v': /* Verbose */ + break; + case 'v': /* Verbose */ + if (argument == disabled_my_option) + { + check_param.testflag&= ~T_VERBOSE; + check_param.verbose=0; + } + else + { check_param.testflag|= T_VERBOSE; check_param.verbose++; - break; - case 'O': - if (set_changeable_var(optarg, changeable_vars)) - { - usage(); - exit(1); - } - break; - case 'R': /* Sort records */ - old_testflag=check_param.testflag; + } + break; + case 'R': /* Sort records */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_SORT_RECORDS; + else + { check_param.testflag|= T_SORT_RECORDS; - check_param.opt_sort_key=(uint) atoi(optarg)-1; + check_param.opt_sort_key= (uint) atoi(argument) - 1; if (check_param.opt_sort_key >= MI_MAX_KEY) { fprintf(stderr, @@ -417,51 +596,86 @@ static void get_options(register int *argc,register char ***argv) MI_MAX_KEY); exit(1); } - break; - case 'S': /* Sort index */ - old_testflag=check_param.testflag; + } + break; + case 'S': /* Sort index */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_SORT_INDEX; + else check_param.testflag|= T_SORT_INDEX; - break; - case 't': - check_param.tmpdir=optarg; - break; - case 'T': + break; + case 'T': + if (argument == disabled_my_option) + check_param.testflag&= ~T_READONLY; + else check_param.testflag|= T_READONLY; - break; - case 'U': + break; + case 'U': + if (argument == disabled_my_option) + check_param.testflag&= ~T_UPDATE_STATE; + else check_param.testflag|= T_UPDATE_STATE; - break; - case '#': - DBUG_PUSH(optarg ? optarg : "d:t:o,/tmp/myisamchk.trace"); - break; - case 'V': - print_version(); - exit(0); - case OPT_CHARSETS_DIR: - charsets_dir = optarg; - break; - case OPT_SET_CHARSET: - set_charset_name=optarg; - break; + break; + case '#': + if (argument == disabled_my_option) + { + DBUG_POP(); + } + else + { + DBUG_PUSH(argument ? argument : "d:t:o,/tmp/myisamchk.trace"); + } + break; + case 'V': + print_version(); + exit(0); + case OPT_CORRECT_CHECKSUM: + if (argument == disabled_my_option) + check_param.testflag&= ~T_CALC_CHECKSUM; + else + check_param.testflag|= T_CALC_CHECKSUM; + break; #ifdef DEBUG /* Only useful if debugging */ - case OPT_START_CHECK_POS: - check_param.start_check_pos=strtoull(optarg,NULL,0); - break; + case OPT_START_CHECK_POS: + check_param.start_check_pos= strtoull(argument, NULL, 0); + break; #endif - case '?': - usage(); - exit(0); - } + case 'H': + my_print_help(my_long_options); + exit(0); + case '?': + usage(); + exit(0); } - (*argc)-=optind; - (*argv)+=optind; + return 0; +} + + +static void get_options(register int *argc,register char ***argv) +{ + int ho_error; + + load_defaults("my", load_default_groups, argc, argv); + default_argv= *argv; + if (isatty(fileno(stdout))) + check_param.testflag|=T_WRITE_LOOP; + + if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option))) + exit(ho_error); + + /* If using repair, then update checksum if one uses --update-state */ + if ((check_param.testflag & T_UPDATE_STATE) && + (check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_CALC_CHECKSUM; + if (*argc == 0) { usage(); exit(-1); } + if ((check_param.testflag & T_UNPACK) && - (check_param.opt_rep_quick || (check_param.testflag & T_SORT_RECORDS))) + (check_param.testflag & (T_QUICK | T_SORT_RECORDS))) { VOID(fprintf(stderr, "%s: --unpack can't be used with --quick or --sort-records\n", @@ -470,7 +684,7 @@ static void get_options(register int *argc,register char ***argv) } if ((check_param.testflag & T_READONLY) && (check_param.testflag & - (T_REP_BY_SORT | T_REP | T_STATISTICS | T_AUTO_INC | + (T_REP_ANY | T_STATISTICS | T_AUTO_INC | T_SORT_RECORDS | T_SORT_INDEX | T_FORCE_CREATE))) { VOID(fprintf(stderr, @@ -483,6 +697,7 @@ static void get_options(register int *argc,register char ***argv) if (!(set_charset=get_charset_by_name(set_charset_name, MYF(MY_WME)))) exit(1); + myisam_block_size=(uint) 1 << my_bit_log2(opt_myisam_block_size); return; } /* get options */ @@ -492,11 +707,10 @@ static void get_options(register int *argc,register char ***argv) static int myisamchk(MI_CHECK *param, my_string filename) { int error,lock_type,recreate; - int rep_quick= param->opt_rep_quick; + int rep_quick= param->testflag & (T_QUICK | T_FORCE_UNIQUENESS); uint raid_chunks; MI_INFO *info; File datafile; - char fixed_name[FN_REFLEN]; char llbuff[22],llbuff2[22]; my_bool state_updated=0; MYISAM_SHARE *share; @@ -521,7 +735,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) case HA_ERR_CRASHED: mi_check_print_error(param,"'%s' doesn't have a correct index definition. You need to recreate it before you can do a repair",filename); break; - case HA_ERR_WRONG_TABLE_DEF: + case HA_ERR_NOT_A_TABLE: mi_check_print_error(param,"'%s' is not a MyISAM-table",filename); break; case HA_ERR_CRASHED_ON_USAGE: @@ -554,11 +768,12 @@ static int myisamchk(MI_CHECK *param, my_string filename) } share=info->s; share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */ + share->tot_locks-= share->r_locks; share->r_locks=0; raid_chunks=share->base.raid_chunks; /* - Skipp the checking of the file if: + Skip the checking of the file if: We are using --fast and the table is closed properly We are using --check-only-changed-tables and the table hasn't changed */ @@ -566,7 +781,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) { my_bool need_to_check= mi_is_crashed(info) || share->state.open_count != 0; - if ((param->testflag & (T_REP_BY_SORT | T_REP | T_SORT_RECORDS)) && + if ((param->testflag & (T_REP_ANY | T_SORT_RECORDS)) && ((share->state.changed & (STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR) || !(param->testflag & T_CHECK_ONLY_CHANGED)))) @@ -598,7 +813,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) DBUG_RETURN(0); } } - if ((param->testflag & (T_REP_BY_SORT | T_REP | T_STATISTICS | + if ((param->testflag & (T_REP_ANY | T_STATISTICS | T_SORT_RECORDS | T_SORT_INDEX)) && (((param->testflag & T_UNPACK) && share->data_file_type == COMPRESSED_RECORD) || @@ -610,11 +825,12 @@ static int myisamchk(MI_CHECK *param, my_string filename) (((ulonglong) 1L << share->base.keys)-1)) || test_if_almost_full(info) || info->s->state.header.file_version[3] != myisam_file_magic[3] || - (set_charset && set_charset->number != share->state.header.language))) + (set_charset && set_charset->number != share->state.header.language) || + myisam_block_size != MI_KEY_BLOCK_LENGTH)) { if (set_charset) - check_param.language=set_charset->number; - if (recreate_table(&check_param, &info,filename)) + param->language=set_charset->number; + if (recreate_table(param, &info,filename)) { VOID(fprintf(stderr, "MyISAM-table '%s' is not fixed because of errors\n", @@ -622,15 +838,15 @@ static int myisamchk(MI_CHECK *param, my_string filename) return(-1); } recreate=1; - if (!(param->testflag & (T_REP | T_REP_BY_SORT))) + if (!(param->testflag & T_REP_ANY)) { param->testflag|=T_REP_BY_SORT; /* if only STATISTICS */ if (!(param->testflag & T_SILENT)) printf("- '%s' has old table-format. Recreating index\n",filename); - if (!rep_quick) - rep_quick=1; + rep_quick|=T_QUICK; } share=info->s; + share->tot_locks-= share->r_locks; share->r_locks=0; } @@ -639,12 +855,12 @@ static int myisamchk(MI_CHECK *param, my_string filename) param->total_files++; param->total_records+=info->state->records; param->total_deleted+=info->state->del; - descript(&check_param, info, filename); + descript(param, info, filename); } else { if (share->fulltext_index) - ft_init_stopwords(ft_precompiled_stopwords); /* SerG */ + ft_init_stopwords(); if (!(param->testflag & T_READONLY)) lock_type = F_WRLCK; /* table is changed */ @@ -660,13 +876,14 @@ static int myisamchk(MI_CHECK *param, my_string filename) goto end2; } share->w_locks++; /* Mark for writeinfo */ + share->tot_locks++; info->lock_type= F_EXTRA_LCK; /* Simulate as locked */ info->tmp_lock_type=lock_type; datafile=info->dfile; - if (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX)) + if (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)) { - if (param->testflag & (T_REP+T_REP_BY_SORT)) + if (param->testflag & T_REP_ANY) { ulonglong tmp=share->state.key_map; share->state.key_map= (((ulonglong) 1 << share->base.keys)-1) @@ -674,11 +891,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) if (tmp != share->state.key_map) info->update|=HA_STATE_CHANGED; } - VOID(fn_format(fixed_name,filename,"",MI_NAME_IEXT, - 4+ (param->opt_follow_links ? 16 : 0))); - - if (rep_quick && chk_del(&check_param, info, - param->testflag & ~T_VERBOSE)) + if (rep_quick && chk_del(param, info, param->testflag & ~T_VERBOSE)) { if (param->testflag & T_FORCE_CREATE) { @@ -694,18 +907,21 @@ static int myisamchk(MI_CHECK *param, my_string filename) } if (!error) { - if ((param->testflag & T_REP_BY_SORT) && + if ((param->testflag & (T_REP_BY_SORT | T_REP_PARALLEL)) && (share->state.key_map || (rep_quick && !param->keys_in_use && !recreate)) && mi_test_if_sort_rep(info, info->state->records, info->s->state.key_map, - check_param.force_sort)) + param->force_sort)) { - error=mi_repair_by_sort(&check_param,info,fixed_name,rep_quick); + if (param->testflag & T_REP_BY_SORT) + error=mi_repair_by_sort(param,info,filename,rep_quick); + else + error=mi_repair_parallel(param,info,filename,rep_quick); state_updated=1; } - else if (param->testflag & (T_REP | T_REP_BY_SORT)) - error=mi_repair(&check_param, info,fixed_name,rep_quick); + else if (param->testflag & T_REP_ANY) + error=mi_repair(param, info,filename,rep_quick); } if (!error && param->testflag & T_SORT_RECORDS) { @@ -717,10 +933,10 @@ static int myisamchk(MI_CHECK *param, my_string filename) if (param->out_flag & O_NEW_DATA) { /* Change temp file to org file */ VOID(my_close(info->dfile,MYF(MY_WME))); /* Close new file */ - error|=change_to_newfile(fixed_name,MI_NAME_DEXT,DATA_TMP_EXT, + error|=change_to_newfile(filename,MI_NAME_DEXT,DATA_TMP_EXT, raid_chunks, MYF(0)); - if (mi_open_datafile(info,info->s)) + if (mi_open_datafile(info,info->s, -1)) error=1; param->out_flag&= ~O_NEW_DATA; /* We are using new datafile */ param->read_cache.file=info->dfile; @@ -731,27 +947,28 @@ static int myisamchk(MI_CHECK *param, my_string filename) uint key; /* We can't update the index in mi_sort_records if we have a - prefix compressed index + prefix compressed or fulltext index */ my_bool update_index=1; for (key=0 ; key < share->base.keys; key++) - if (share->keyinfo[key].flag & HA_BINARY_PACK_KEY) + if (share->keyinfo[key].flag & (HA_BINARY_PACK_KEY|HA_FULLTEXT)) update_index=0; - error=mi_sort_records(param,info,fixed_name,param->opt_sort_key, + error=mi_sort_records(param,info,filename,param->opt_sort_key, + /* what is the following parameter for ? */ (my_bool) !(param->testflag & T_REP), update_index); datafile=info->dfile; /* This is now locked */ if (!error && !update_index) { - if (check_param.verbose) + if (param->verbose) puts("Table had a compressed index; We must now recreate the index"); - error=mi_repair_by_sort(&check_param,info,fixed_name,1); + error=mi_repair_by_sort(param,info,filename,1); } } } if (!error && param->testflag & T_SORT_INDEX) - error=mi_sort_index(param,info,fixed_name); + error=mi_sort_index(param,info,filename); if (!error) share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR); @@ -786,7 +1003,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) !(param->testflag & (T_FAST | T_FORCE_CREATE))) { if (param->testflag & (T_EXTEND | T_MEDIUM)) - VOID(init_key_cache(param->use_buffers,(uint) NEAD_MEM)); + VOID(init_key_cache(param->use_buffers)); VOID(init_io_cache(¶m->read_cache,datafile, (uint) param->read_buffer_length, READ_CACHE, @@ -820,8 +1037,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) } } if ((param->testflag & T_AUTO_INC) || - ((param->testflag & (T_REP | T_REP_BY_SORT)) && - info->s->base.auto_key)) + ((param->testflag & T_REP_ANY) && info->s->base.auto_key)) update_auto_increment_key(param, info, (my_bool) !test(param->testflag & T_AUTO_INC)); @@ -830,7 +1046,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) if (info->update & HA_STATE_CHANGED && ! (param->testflag & T_READONLY)) error|=update_state_info(param, info, UPDATE_OPEN_COUNT | - (((param->testflag & (T_REP | T_REP_BY_SORT)) ? + (((param->testflag & T_REP_ANY) ? UPDATE_TIME : 0) | (state_updated ? UPDATE_STAT : 0) | ((param->testflag & T_SORT_RECORDS) ? @@ -839,6 +1055,7 @@ static int myisamchk(MI_CHECK *param, my_string filename) info->update&= ~HA_STATE_CHANGED; } share->w_locks--; + share->tot_locks--; end2: if (mi_close(info)) { @@ -848,23 +1065,23 @@ end2: if (error == 0) { if (param->out_flag & O_NEW_DATA) - error|=change_to_newfile(fixed_name,MI_NAME_DEXT,DATA_TMP_EXT, + error|=change_to_newfile(filename,MI_NAME_DEXT,DATA_TMP_EXT, raid_chunks, ((param->testflag & T_BACKUP_DATA) ? MYF(MY_REDEL_MAKE_BACKUP) : MYF(0))); if (param->out_flag & O_NEW_INDEX) - error|=change_to_newfile(fixed_name,MI_NAME_IEXT,INDEX_TMP_EXT,0, + error|=change_to_newfile(filename,MI_NAME_IEXT,INDEX_TMP_EXT,0, MYF(0)); } VOID(fflush(stdout)); VOID(fflush(stderr)); if (param->error_printed) { - if (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX)) + if (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)) { VOID(fprintf(stderr, "MyISAM-table '%s' is not fixed because of errors\n", filename)); - if (check_param.testflag & (T_REP_BY_SORT | T_REP)) + if (param->testflag & T_REP_ANY) VOID(fprintf(stderr, "Try fixing it by using the --safe-recover (-o) or the --force (-f) option\n")); } @@ -875,7 +1092,7 @@ end2: filename)); } else if (param->warning_printed && - ! (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX+ + ! (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX | T_FORCE_CREATE))) VOID(fprintf(stderr, "MyISAM-table '%s' is usable but should be fixed\n", filename)); @@ -1155,7 +1372,7 @@ static int mi_sort_records(MI_CHECK *param, register MI_INFO *info, my_string name, uint sort_key, my_bool write_info, - my_bool update_index) + my_bool update_index) { int got_error; uint key; @@ -1165,11 +1382,14 @@ static int mi_sort_records(MI_CHECK *param, ha_rows old_record_count; MYISAM_SHARE *share=info->s; char llbuff[22],llbuff2[22]; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO sort_info; + MI_SORT_PARAM sort_param; DBUG_ENTER("sort_records"); - bzero((char*) sort_info,sizeof(*sort_info)); - sort_info->param=param; + bzero((char*)&sort_info,sizeof(sort_info)); + bzero((char*)&sort_param,sizeof(sort_param)); + sort_param.sort_info=&sort_info; + sort_info.param=param; keyinfo= &share->keyinfo[sort_key]; got_error=1; temp_buff=0; @@ -1182,6 +1402,13 @@ static int mi_sort_records(MI_CHECK *param, param->error_printed=0; DBUG_RETURN(-1); } + if (keyinfo->flag & HA_FULLTEXT) + { + mi_check_print_error(param,"Can't sort table '%s' on FULLTEXT key %d", + name,sort_key+1); + param->error_printed=0; + DBUG_RETURN(-1); + } if (!(param->testflag & T_SILENT)) { printf("- Sorting records for MyISAM-table '%s'\n",name); @@ -1193,7 +1420,7 @@ static int mi_sort_records(MI_CHECK *param, if (share->state.key_root[sort_key] == HA_OFFSET_ERROR) DBUG_RETURN(0); /* Nothing to do */ - init_key_cache(param->use_buffers,NEAD_MEM); + init_key_cache(param->use_buffers); if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length, WRITE_CACHE,share->pack.header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL))) @@ -1205,13 +1432,15 @@ static int mi_sort_records(MI_CHECK *param, mi_check_print_error(param,"Not enough memory for key block"); goto err; } - if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, + if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength, MYF(0)))) { mi_check_print_error(param,"Not enough memory for record"); goto err; } - new_file=my_raid_create(fn_format(param->temp_filename,name,"", + fn_format(param->temp_filename,name,"", MI_NAME_DEXT,2+4+32); + new_file=my_raid_create(fn_format(param->temp_filename, + param->temp_filename,"", DATA_TMP_EXT,2+4), 0,param->tmpfile_createflag, share->base.raid_type, @@ -1245,18 +1474,19 @@ static int mi_sort_records(MI_CHECK *param, } /* Setup param for sort_write_record */ - sort_info->info=info; - sort_info->new_data_file_type=share->data_file_type; - sort_info->fix_datafile=1; - sort_info->filepos=share->pack.header_length; + sort_info.info=info; + sort_info.new_data_file_type=share->data_file_type; + sort_param.fix_datafile=1; + sort_param.master=1; + sort_param.filepos=share->pack.header_length; old_record_count=info->state->records; info->state->records=0; - if (sort_info->new_data_file_type != COMPRESSED_RECORD) + if (sort_info.new_data_file_type != COMPRESSED_RECORD) share->state.checksum=0; - if (sort_record_index(param, info,keyinfo,share->state.key_root[sort_key], + if (sort_record_index(&sort_param,info,keyinfo,share->state.key_root[sort_key], temp_buff, sort_key,new_file,update_index) || - write_data_suffix(param, info) || + write_data_suffix(&sort_info,1) || flush_io_cache(&info->rec_cache)) goto err; @@ -1274,7 +1504,7 @@ static int mi_sort_records(MI_CHECK *param, info->state->del=0; info->state->empty=0; share->state.dellink= HA_OFFSET_ERROR; - info->state->data_file_length=sort_info->filepos; + info->state->data_file_length=sort_param.filepos; share->state.split=info->state->records; /* Only hole records */ share->state.version=(ulong) time((time_t*) 0); @@ -1298,11 +1528,11 @@ err: { my_afree((gptr) temp_buff); } - my_free(sort_info->record,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR)); info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); VOID(end_io_cache(&info->rec_cache)); - my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); - sort_info->buff=0; + my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR)); + sort_info.buff=0; share->state.sortkey=sort_key; DBUG_RETURN(flush_blocks(param, share->kfile) | got_error); } /* sort_records */ @@ -1310,7 +1540,8 @@ err: /* Sort records recursive using one index */ -static int sort_record_index(MI_CHECK *param,MI_INFO *info, MI_KEYDEF *keyinfo, +static int sort_record_index(MI_SORT_PARAM *sort_param,MI_INFO *info, + MI_KEYDEF *keyinfo, my_off_t page, uchar *buff, uint sort_key, File new_file,my_bool update_index) { @@ -1319,7 +1550,8 @@ static int sort_record_index(MI_CHECK *param,MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t next_page,rec_pos; uchar lastkey[MI_MAX_KEY_BUFF]; char llbuff[22]; - SORT_INFO *sort_info= ¶m->sort_info; + SORT_INFO *sort_info= sort_param->sort_info; + MI_CHECK *param=sort_info->param; DBUG_ENTER("sort_record_index"); nod_flag=mi_test_if_nod(buff); @@ -1350,7 +1582,7 @@ static int sort_record_index(MI_CHECK *param,MI_INFO *info, MI_KEYDEF *keyinfo, llstr(next_page,llbuff)); goto err; } - if (sort_record_index(param, info,keyinfo,next_page,temp_buff,sort_key, + if (sort_record_index(sort_param, info,keyinfo,next_page,temp_buff,sort_key, new_file, update_index)) goto err; } @@ -1361,23 +1593,23 @@ static int sort_record_index(MI_CHECK *param,MI_INFO *info, MI_KEYDEF *keyinfo, break; rec_pos= _mi_dpos(info,0,lastkey+key_length); - if ((*info->s->read_rnd)(info,sort_info->record,rec_pos,0)) + if ((*info->s->read_rnd)(info,sort_param->record,rec_pos,0)) { mi_check_print_error(param,"%d when reading datafile",my_errno); goto err; } - if (rec_pos != sort_info->filepos && update_index) + if (rec_pos != sort_param->filepos && update_index) { _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength, - sort_info->filepos); - if (movepoint(info,sort_info->record,rec_pos,sort_info->filepos, + sort_param->filepos); + if (movepoint(info,sort_param->record,rec_pos,sort_param->filepos, sort_key)) { mi_check_print_error(param,"%d when updating key-pointers",my_errno); goto err; } } - if (sort_write_record(sort_info)) + if (sort_write_record(sort_param)) goto err; } /* Clear end of block to get better compression if the table is backuped */ @@ -1449,7 +1681,7 @@ void mi_check_print_error(MI_CHECK *param, const char *fmt,...) if (!param->warning_printed && !param->error_printed) { if (param->testflag & T_SILENT) - fprintf(stderr,"%s: ISAM file %s\n",my_progname,param->isam_file_name); + fprintf(stderr,"%s: MyISAM file %s\n",my_progname,param->isam_file_name); param->out_flag|= O_DATA_LOST; } param->error_printed|=1; diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h index 260e9665b31..92eead7b96c 100644 --- a/myisam/myisamdef.h +++ b/myisam/myisamdef.h @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -18,6 +18,7 @@ #include "myisam.h" /* Structs & some defines */ #include "myisampack.h" /* packing of keys */ +#include <my_tree.h> #ifdef THREAD #include <my_pthread.h> #include <thr_lock.h> @@ -37,7 +38,7 @@ typedef struct st_mi_status_info my_off_t key_empty; /* lost space in indexfile */ my_off_t key_file_length; my_off_t data_file_length; -} MI_STATUS_INFO; +} MI_STATUS_INFO; typedef struct st_mi_state_info { @@ -65,8 +66,10 @@ typedef struct st_mi_state_info ulong unique; /* Unique number for this process */ ulong update_count; /* Updated for each write lock */ ulong status; + ulong *rec_per_key_part; my_off_t *key_root; /* Start of key trees */ my_off_t *key_del; /* delete links for trees */ + my_off_t rec_per_key_rows; /* Rows when calculating rec_per_key */ ulong sec_index_changed; /* Updated when new sec_index */ ulong sec_index_used; /* which extra index are in use */ @@ -79,8 +82,6 @@ typedef struct st_mi_state_info uint sortkey; /* sorted by this key (not used) */ uint open_count; uint8 changed; /* Changed since myisamchk */ - my_off_t rec_per_key_rows; /* Rows when calculating rec_per_key */ - ulong *rec_per_key_part; /* the following isn't saved on disk */ uint state_diff_length; /* Should be 0 */ @@ -159,42 +160,45 @@ typedef struct st_mi_isam_share { /* Shared between opens */ MI_COLUMNDEF *rec; /* Pointer to field information */ MI_PACK pack; /* Data about packed records */ MI_BLOB *blobs; /* Pointer to blobs */ - char *filename; /* Name of indexfile */ + char *unique_file_name; /* realpath() of index file */ + char *data_file_name, /* Resolved path names from symlinks */ + *index_file_name; byte *file_map; /* mem-map of file if possible */ + MI_DECODE_TREE *decode_trees; + uint16 *decode_tables; + int (*read_record)(struct st_myisam_info*, my_off_t, byte*); + int (*write_record)(struct st_myisam_info*, const byte*); + int (*update_record)(struct st_myisam_info*, my_off_t, const byte*); + int (*delete_record)(struct st_myisam_info*); + int (*read_rnd)(struct st_myisam_info*, byte*, my_off_t, my_bool); + int (*compare_record)(struct st_myisam_info*, const byte *); + ha_checksum (*calc_checksum)(struct st_myisam_info*, const byte *); + int (*compare_unique)(struct st_myisam_info*, MI_UNIQUEDEF *, + const byte *record, my_off_t pos); + invalidator_by_filename invalidator; /* query cache invalidator */ ulong this_process; /* processid */ ulong last_process; /* For table-change-check */ ulong last_version; /* Version on start */ ulong options; /* Options used */ + ulong min_pack_length; /* Theese are used by packed data */ + ulong max_pack_length; + ulong state_diff_length; uint rec_reflength; /* rec_reflength in use now */ - int kfile; /* Shared keyfile */ - int data_file; /* Shared data file */ + File kfile; /* Shared keyfile */ + File data_file; /* Shared data file */ int mode; /* mode of file on open */ uint reopen; /* How many times reopened */ - uint w_locks,r_locks; /* Number of read/write locks */ + uint w_locks,r_locks,tot_locks; /* Number of read/write locks */ uint blocksize; /* blocksize of keyfile */ - ulong min_pack_length; /* Theese are used by packed data */ - ulong max_pack_length; - ulong state_diff_length; + myf write_flag; + int rnd; /* rnd-counter */ + enum data_file_type data_file_type; my_bool changed, /* If changed since lock */ global_changed, /* If changed since open */ not_flushed, temporary,delay_key_write, concurrent_insert, fulltext_index; - myf write_flag; - int rnd; /* rnd-counter */ - MI_DECODE_TREE *decode_trees; - uint16 *decode_tables; - enum data_file_type data_file_type; - int (*read_record)(struct st_myisam_info*, my_off_t, byte*); - int (*write_record)(struct st_myisam_info*, const byte*); - int (*update_record)(struct st_myisam_info*, my_off_t, const byte*); - int (*delete_record)(struct st_myisam_info*); - int (*read_rnd)(struct st_myisam_info*, byte*, my_off_t, my_bool); - int (*compare_record)(struct st_myisam_info*, const byte *); - ha_checksum (*calc_checksum)(struct st_myisam_info*, const byte *); - int (*compare_unique)(struct st_myisam_info*, MI_UNIQUEDEF *, - const byte *record, my_off_t pos); #ifdef THREAD THR_LOCK lock; pthread_mutex_t intern_lock; /* Locking for use with _locking */ @@ -212,16 +216,21 @@ typedef struct st_mi_bit_buff { /* Used for packing of record */ uint error; } MI_BIT_BUFF; - struct st_myisam_info { MYISAM_SHARE *s; /* Shared between open:s */ MI_STATUS_INFO *state,save_state; MI_BLOB *blobs; /* Pointer to blobs */ - int dfile; /* The datafile */ - MI_BIT_BUFF bit_buff; - uint opt_flag; /* Optim. for space/speed */ - uint update; /* If file changed since open */ + MI_BIT_BUFF bit_buff; + /* accumulate indexfile changes between write's */ + TREE *bulk_insert; char *filename; /* parameter to open filename */ + uchar *buff, /* Temp area for key */ + *lastkey,*lastkey2; /* Last used search key */ + byte *rec_buff; /* Tempbuff for recordpack */ + uchar *int_keypos, /* Save position for next/previous */ + *int_maxpos; /* -""- */ + int (*read_record)(struct st_myisam_info*, my_off_t, byte*); + invalidator_by_filename invalidator; /* query cache invalidator */ ulong this_unique; /* uniq filenumber or thread */ ulong last_unique; /* last unique number */ ulong this_loop; /* counter for this open */ @@ -230,20 +239,15 @@ struct st_myisam_info { nextpos; /* Position to next record */ my_off_t save_lastpos; my_off_t pos; /* Intern variable */ - ha_checksum checksum; - ulong packed_length,blob_length; /* Length of found, packed record */ - uint alloced_rec_buff_length; /* Max recordlength malloced */ - uchar *buff, /* Temp area for key */ - *lastkey,*lastkey2; /* Last used search key */ - byte *rec_buff, /* Tempbuff for recordpack */ - *rec_alloc; /* Malloced area for record */ - uchar *int_keypos, /* Save position for next/previous */ - *int_maxpos; /* -""- */ - uint32 int_keytree_version; /* -""- */ - uint int_nod_flag; /* -""- */ my_off_t last_keypage; /* Last key page read */ my_off_t last_search_keypage; /* Last keypage when searching */ my_off_t dupp_key_pos; + ha_checksum checksum; + ulong packed_length,blob_length; /* Length of found, packed record */ + int dfile; /* The datafile */ + uint opt_flag; /* Optim. for space/speed */ + uint update; /* If file changed since open */ + uint int_nod_flag; /* -""- */ int lastinx; /* Last used index */ uint lastkey_length; /* Length of key in lastkey */ uint last_rkey_length; /* Last length in mi_rkey() */ @@ -254,15 +258,15 @@ struct st_myisam_info { uint data_changed; /* Somebody has changed data */ uint save_update; /* When using KEY_READ */ int save_lastinx; + uint32 int_keytree_version; /* -""- */ + LIST open_list; + IO_CACHE rec_cache; /* When cacheing records */ + myf lock_wait; /* is 0 or MY_DONT_WAIT */ my_bool was_locked; /* Was locked in panic */ my_bool quick_mode; my_bool page_changed; /* If info->buff can't be used for rnext */ my_bool buff_used; /* If info->buff has to be reread for rnext */ my_bool use_packed_key; /* For MYISAMMRG */ - myf lock_wait; /* is 0 or MY_DONT_WAIT */ - int (*read_record)(struct st_myisam_info*, my_off_t, byte*); - LIST open_list; - IO_CACHE rec_cache; /* When cacheing records */ #ifdef THREAD THR_LOCK_DATA lock; #endif @@ -285,7 +289,7 @@ struct st_myisam_info { #define STATE_CHANGED 1 #define STATE_CRASHED 2 -#define STATE_CRASHED_ON_REPAIR 4 +#define STATE_CRASHED_ON_REPAIR 4 #define STATE_NOT_ANALYZED 8 #define STATE_NOT_OPTIMIZED_KEYS 16 #define STATE_NOT_SORTED_PAGES 32 @@ -352,7 +356,9 @@ struct st_myisam_info { #define MI_DYN_MAX_BLOCK_LENGTH ((1L << 24)-4L) #define MI_DYN_MAX_ROW_LENGTH (MI_DYN_MAX_BLOCK_LENGTH - MI_SPLIT_LENGTH) #define MI_DYN_ALIGN_SIZE 4 /* Align blocks on this */ -#define MI_MAX_DYN_HEADER_BYTE 12 /* max header byte for dynamic rows */ +#define MI_MAX_DYN_HEADER_BYTE 13 /* max header byte for dynamic rows */ +#define MI_MAX_BLOCK_LENGTH ((((ulong) 1 << 24)-1) & (~ (ulong) (MI_DYN_ALIGN_SIZE-1))) +#define MI_REC_BUFF_OFFSET ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32)) #define MEMMAP_EXTRA_MARGIN 7 /* Write this as a suffix for file */ @@ -361,12 +367,13 @@ struct st_myisam_info { #define PACK_TYPE_ZERO_FILL 4 #define MI_FOUND_WRONG_KEY 32738 /* Impossible value from _mi_key_cmp */ -#define MI_KEY_BLOCK_LENGTH 1024 /* Min key block length */ -#define MI_MAX_KEY_BLOCK_LENGTH 8192 -#define MI_MAX_KEY_BLOCK_SIZE (MI_MAX_KEY_BLOCK_LENGTH/MI_KEY_BLOCK_LENGTH) -#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer) ((((key_length+data_pointer+key_pointer)*4+key_pointer+2)/MI_KEY_BLOCK_LENGTH+1)*MI_KEY_BLOCK_LENGTH) -#define MI_MAX_KEYPTR_SIZE 5 /* For calculating block lengths */ -#define MI_MIN_KEYBLOCK_LENGTH 50 /* When to split delete blocks */ +#define MI_MAX_KEY_BLOCK_SIZE (MI_MAX_KEY_BLOCK_LENGTH/MI_MIN_KEY_BLOCK_LENGTH) +#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer) ((((key_length+data_pointer+key_pointer)*4+key_pointer+2)/myisam_block_size+1)*myisam_block_size) +#define MI_MAX_KEYPTR_SIZE 5 /* For calculating block lengths */ +#define MI_MIN_KEYBLOCK_LENGTH 50 /* When to split delete blocks */ + +#define MI_MIN_SIZE_BULK_INSERT_TREE 16384 /* this is per key */ +#define MI_MIN_ROWS_TO_USE_BULK_INSERT 100 /* The UNIQUE check is done with a hashed long key */ @@ -473,6 +480,9 @@ extern int _mi_bin_search(struct st_myisam_info *info,MI_KEYDEF *keyinfo, extern int _mi_seq_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *page, uchar *key,uint key_len,uint comp_flag, uchar **ret_pos,uchar *buff, my_bool *was_last_key); +extern int _mi_prefix_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *page, + uchar *key,uint key_len,uint comp_flag, + uchar **ret_pos,uchar *buff, my_bool *was_last_key); extern int _mi_compare_text(CHARSET_INFO *, uchar *, uint, uchar *, uint , my_bool); extern my_off_t _mi_kpos(uint nod_flag,uchar *after_key); @@ -513,10 +523,17 @@ extern int _mi_read_key_record(MI_INFO *info,my_off_t filepos,byte *buf); extern int _mi_read_cache(IO_CACHE *info,byte *buff,my_off_t pos, uint length,int re_read_if_possibly); extern void update_auto_increment(MI_INFO *info,const byte *record); -extern byte *mi_fix_rec_buff_for_blob(MI_INFO *info,ulong blob_length); + +extern byte *mi_alloc_rec_buff(MI_INFO *,ulong, byte**); +#define mi_get_rec_buff_ptr(info,buf) \ + ((((info)->s->options & HA_OPTION_PACK_RECORD) && (buf)) ? \ + (buf) - MI_REC_BUFF_OFFSET : (buf)) +#define mi_get_rec_buff_len(info,buf) \ + (*((uint32 *)(mi_get_rec_buff_ptr(info,buf)))) + extern ulong _mi_rec_unpack(MI_INFO *info,byte *to,byte *from, ulong reclength); -extern my_bool _mi_rec_check(MI_INFO *info,const char *from); +extern my_bool _mi_rec_check(MI_INFO *info,const char *record, byte *packpos); extern int _mi_write_part_record(MI_INFO *info,my_off_t filepos,ulong length, my_off_t next_filepos,byte **record, ulong *reclength,int *flag); @@ -528,6 +545,8 @@ extern int _mi_read_rnd_pack_record(MI_INFO*, byte *,my_off_t, my_bool); extern int _mi_pack_rec_unpack(MI_INFO *info,byte *to,byte *from, ulong reclength); extern ulonglong mi_safe_mul(ulonglong a,ulonglong b); +extern int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, + const byte *oldrec, const byte *newrec, my_off_t pos); struct st_sort_info; @@ -554,7 +573,7 @@ typedef struct st_mi_block_info { /* Parameter to _mi_get_block_info */ #define BLOCK_SYNC_ERROR 16 /* Right data at wrong place */ #define BLOCK_FATAL_ERROR 32 /* hardware-error */ -#define NEAD_MEM ((uint) 10*4*(IO_SIZE+32)+32) /* Nead for recursion */ +#define NEED_MEM ((uint) 10*4*(IO_SIZE+32)+32) /* Nead for recursion */ #define MAXERR 20 #define BUFFERS_WHEN_SORTING 16 /* Alloc for sort-key-tree */ #define WRITE_COUNT MY_HOW_OFTEN_TO_WRITE @@ -580,14 +599,16 @@ enum myisam_log_commands { #define myisam_log_command(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_command(a,b,c,d,e) #define myisam_log_record(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_record(a,b,c,d,e) +#define fast_mi_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _mi_writeinfo((INFO),0) +#define fast_mi_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _mi_readinfo((INFO),F_RDLCK,1) + #ifdef __cplusplus extern "C" { #endif extern uint _mi_get_block_info(MI_BLOCK_INFO *,File, my_off_t); extern uint _mi_rec_pack(MI_INFO *info,byte *to,const byte *from); -extern uint _mi_pack_get_block_info(MI_INFO *mysql, MI_BLOCK_INFO *, File, - my_off_t, char *rec_buf); +extern uint _mi_pack_get_block_info(MI_INFO *, MI_BLOCK_INFO *, File, my_off_t); extern void _my_store_blob_length(byte *pos,uint pack_length,uint length); extern void _myisam_log(enum myisam_log_commands command,MI_INFO *info, const byte *buffert,uint length); @@ -633,14 +654,19 @@ my_bool mi_check_status(void* param); void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows); my_bool check_table_is_closed(const char *name, const char *where); -int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share); +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share, File file_to_dup); int mi_open_keyfile(MYISAM_SHARE *share); void mi_setup_functions(register MYISAM_SHARE *share); -/* Functions needed by mi_check */ + /* Functions needed by mi_check */ void mi_check_print_error _VARARGS((MI_CHECK *param, const char *fmt,...)); void mi_check_print_warning _VARARGS((MI_CHECK *param, const char *fmt,...)); void mi_check_print_info _VARARGS((MI_CHECK *param, const char *fmt,...)); +int flush_pending_blocks(MI_SORT_PARAM *param); +int thr_write_keys(MI_SORT_PARAM *sort_param); +#ifdef THREAD +pthread_handler_decl(thr_find_all_keys,arg); +#endif #ifdef __cplusplus } diff --git a/myisam/myisamlog.c b/myisam/myisamlog.c index ddfafb91430..ceca8d2a0e3 100644 --- a/myisam/myisamlog.c +++ b/myisam/myisamlog.c @@ -56,7 +56,7 @@ extern int main(int argc,char * *argv); static void get_options(int *argc,char ***argv); static int examine_log(my_string file_name,char **table_names); static int read_string(IO_CACHE *file,gptr *to,uint length); -static int file_info_compare(void *a,void *b); +static int file_info_compare(void *cmp_arg, void *a,void *b); static int test_if_open(struct file_info *key,element_count count, struct test_if_open_param *param); static void fix_blob_pointers(MI_INFO *isam,byte *record); @@ -331,9 +331,9 @@ static int examine_log(my_string file_name, char **table_names) init_io_cache(&cache,file,0,READ_CACHE,start_offset,0,MYF(0)); bzero((gptr) com_count,sizeof(com_count)); - init_tree(&tree,0,sizeof(file_info),(qsort_cmp) file_info_compare,1, - (void(*)(void*)) file_info_free); - VOID(init_key_cache(KEY_CACHE_SIZE,(uint) (10*4*(IO_SIZE+MALLOC_OVERHEAD)))); + init_tree(&tree,0,0,sizeof(file_info),(qsort_cmp2) file_info_compare,1, + (tree_element_free) file_info_free, NULL); + VOID(init_key_cache(KEY_CACHE_SIZE)); files_open=0; access_time=0; while (access_time++ != number_of_commands && @@ -404,11 +404,7 @@ static int examine_log(my_string file_name, char **table_names) } to=isam_file_name; if (filepath) - { - strmov(isam_file_name,filepath); - convert_dirname(isam_file_name); - to=strend(isam_file_name); - } + to=convert_dirname(isam_file_name,filepath,NullS); strmov(to,pos); fn_ext(isam_file_name)[0]=0; /* Remove extension */ } @@ -488,7 +484,7 @@ static int examine_log(my_string file_name, char **table_names) command_name[command], (int) extra_command,result); if (update && curr_file_info && !curr_file_info->closed) { - if (mi_extra(curr_file_info->isam, extra_command) != (int) result) + if (mi_extra(curr_file_info->isam, extra_command, 0) != (int) result) { fflush(stdout); VOID(fprintf(stderr, @@ -695,7 +691,8 @@ static int read_string(IO_CACHE *file, register gptr *to, register uint length) } /* read_string */ -static int file_info_compare(void *a, void *b) +static int file_info_compare(void* cmp_arg __attribute__((unused)), + void *a, void *b) { long lint; diff --git a/myisam/myisampack.c b/myisam/myisampack.c index cd18cebbda0..4fc84ac5657 100644 --- a/myisam/myisampack.c +++ b/myisam/myisampack.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -30,7 +30,7 @@ #ifndef __GNU_LIBRARY__ #define __GNU_LIBRARY__ /* Skip warnings in getopt.h */ #endif -#include <getopt.h> +#include <my_getopt.h> #if INT_MAX > 32767 #define BITS_SAVED 32 @@ -124,7 +124,8 @@ static void free_counts_and_tree_and_queue(HUFF_TREE *huff_trees, uint trees, HUFF_COUNTS *huff_counts, uint fields); -static int compare_tree(const uchar *s,const uchar *t); +static int compare_tree(void* cmp_arg __attribute__((unused)), + const uchar *s,const uchar *t); static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts); static void check_counts(HUFF_COUNTS *huff_counts,uint trees, my_off_t records); @@ -168,9 +169,10 @@ static int mrg_rrnd(PACK_MRG_INFO *info,byte *buf); static void mrg_reset(PACK_MRG_INFO *mrg); -static int backup=0,error_on_write=0,test_only=0,verbose=0,silent=0, - write_loop=0,force_pack=0,opt_wait=0,isamchk_neaded=0; +static int error_on_write=0,test_only=0,verbose=0,silent=0, + write_loop=0,force_pack=0, isamchk_neaded=0; static int tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; +static my_bool backup, opt_wait; static uint tree_buff_length=8196-MALLOC_OVERHEAD; static char tmp_dir[FN_REFLEN]={0},*join_table; static my_off_t intervall_length; @@ -231,57 +233,106 @@ int main(int argc, char **argv) enum options_mp {OPT_CHARSETS_DIR_MP=256}; -static struct option long_options[] = +static struct my_option my_long_options[] = { - {"backup", no_argument, 0, 'b'}, - {"character-sets-dir",required_argument,0, OPT_CHARSETS_DIR_MP}, - {"debug", optional_argument, 0, '#'}, - {"force", no_argument, 0, 'f'}, - {"join", required_argument, 0, 'j'}, - {"help", no_argument, 0, '?'}, - {"silent", no_argument, 0, 's'}, - {"tmpdir", required_argument, 0, 'T'}, - {"test", no_argument, 0, 't'}, - {"verbose", no_argument, 0, 'v'}, - {"version", no_argument, 0, 'V'}, - {"wait", no_argument, 0, 'w'}, - {0, 0, 0, 0} + {"backup", 'b', "Make a backup of the table as table_name.OLD", + (gptr*) &backup, (gptr*) &backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"character-sets-dir", OPT_CHARSETS_DIR_MP, + "Directory where character sets are.", (gptr*) &charsets_dir, + (gptr*) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'", + 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"force", 'f', + "Force packing of table even if it gets bigger or if tempfile exists.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"join", 'j', + "Join all given tables into 'new_table_name'. All tables MUST have identical layouts.", + (gptr*) &join_table, (gptr*) &join_table, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', "Be more silent.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 'T', "Use temporary directory to store temporary table.", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"test", 't', "Don't pack table, only test packing it.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Write info about progress and packing result.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', "Output version information and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"wait", 'w', "Wait and retry if table is in use.", (gptr*) &opt_wait, + (gptr*) &opt_wait, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; + static void print_version(void) { - printf("%s Ver 1.13 for %s on %s\n",my_progname,SYSTEM_TYPE,MACHINE_TYPE); + printf("%s Ver 1.22 for %s on %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); } static void usage(void) { print_version(); - puts("Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB"); + puts("Copyright (C) 2002 MySQL AB"); puts("This software comes with ABSOLUTELY NO WARRANTY. This is free software,"); puts("and you are welcome to modify and redistribute it under the GPL license\n"); puts("Pack a MyISAM-table to take much less space."); puts("Keys are not updated, you must run myisamchk -rq on the datafile"); puts("afterwards to update the keys."); - puts("You should give the .MSI file as the filename argument."); + puts("You should give the .MYI file as the filename argument."); printf("\nUsage: %s [OPTIONS] filename...\n", my_progname); - puts("\n\ - -b, --backup Make a backup of the table as table_name.OLD\n\ - -f, --force Force packing of table even if it gets bigger or if\n\ - tempfile exists.\n\ - -j, --join='new_table_name'\n\ - Join all given tables into 'new_table_name'.\n\ - All tables MUST have identical layouts.\n\ - -s, --silent Be more silent.\n\ - -t, --test Don't pack table, only test packing it.\n\ - -v, --verbose Write info about progress and packing result.\n\ - -w, --wait Wait and retry if table is in use.\n\ - -T, --tmpdir=... Use temporary directory to store temporary table.\n\ - -#, --debug=... Output debug log. Often this is 'd:t:o,filename`\n\ - -?, --help Display this help and exit.\n\ - -V, --version Output version information and exit."); - print_defaults("my",load_default_groups); + my_print_help(my_long_options); + print_defaults("my", load_default_groups); + my_print_variables(my_long_options); +} + + +static my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument) +{ + uint length; + + switch(optid) { + case 'f': + force_pack= 1; + tmpfile_createflag= O_RDWR | O_TRUNC; + break; + case 's': + write_loop= verbose= 0; + silent= 1; + break; + case 't': + test_only= verbose= 1; + break; + case 'T': + length= (uint) (strmov(tmp_dir, argument) - tmp_dir); + if (length != dirname_length(tmp_dir)) + { + tmp_dir[length]=FN_LIBCHAR; + tmp_dir[length+1]=0; + } + break; + case 'v': + verbose= 1; + silent= 0; + break; + case '#': + DBUG_PUSH(argument ? argument : "d:t:o"); + break; + case 'V': + print_version(); + exit(0); + case 'I': + case '?': + usage(); + exit(0); + } + return 0; } /* reads options */ @@ -289,66 +340,15 @@ static void usage(void) static void get_options(int *argc,char ***argv) { - int c,option_index=0; - uint length; + int ho_error; my_progname= argv[0][0]; if (isatty(fileno(stdout))) write_loop=1; - while ((c=getopt_long(*argc,*argv,"bfj:stvwT:#::?V",long_options, - &option_index)) != EOF) - { - switch(c) { - case 'b': - backup=1; - break; - case 'f': - force_pack=1; - tmpfile_createflag=O_RDWR | O_TRUNC; - break; - case 'j': - join_table=optarg; - break; - case 's': - write_loop=verbose=0; silent=1; - break; - case 't': - test_only=verbose=1; - break; - case 'T': - length=(uint) (strmov(tmp_dir,optarg)-tmp_dir); - if (length != dirname_length(tmp_dir)) - { - tmp_dir[length]=FN_LIBCHAR; - tmp_dir[length+1]=0; - } - break; - case 'v': - verbose=1; silent=0; - break; - case 'w': - opt_wait=1; - break; - case '#': - DBUG_PUSH(optarg ? optarg : "d:t:o"); - break; - case OPT_CHARSETS_DIR_MP: - charsets_dir = optarg; - break; - case 'V': print_version(); exit(0); - case 'I': - case '?': - usage(); - exit(0); - default: - fprintf(stderr,"%s: Illegal option: -%c\n",my_progname,opterr); - usage(); - exit(1); - } - } - (*argc)-=optind; - (*argv)+=optind; + if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option))) + exit(ho_error); + if (!*argc) { usage(); @@ -663,9 +663,9 @@ static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records) type = FIELD_NORMAL; if (count[i].field_length <= 8 && (type == FIELD_NORMAL || - type == FIELD_SKIPP_ZERO)) + type == FIELD_SKIP_ZERO)) count[i].max_zero_fill= count[i].field_length; - init_tree(&count[i].int_tree,0,-1,(qsort_cmp) compare_tree,0,NULL); + init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree,0,NULL,NULL); if (records && type != FIELD_BLOB && type != FIELD_VARCHAR) count[i].tree_pos=count[i].tree_buff = my_malloc(count[i].field_length > 1 ? tree_buff_length : 2, @@ -787,7 +787,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) /* Save character counters and space-counts and zero-field-counts */ if (count->field_type == FIELD_NORMAL || - count->field_type == FIELD_SKIPP_ENDSPACE) + count->field_type == FIELD_SKIP_ENDSPACE) { for ( ; end_pos > pos ; end_pos--) if (end_pos[-1] != ' ') @@ -806,7 +806,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) count->max_end_space = length; } if (count->field_type == FIELD_NORMAL || - count->field_type == FIELD_SKIPP_PRESPACE) + count->field_type == FIELD_SKIP_PRESPACE) { for (pos=start_pos; pos < end_pos ; pos++) if (pos[0] != ' ') @@ -842,7 +842,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) } if (count->field_length <= 8 && (count->field_type == FIELD_NORMAL || - count->field_type == FIELD_SKIPP_ZERO)) + count->field_type == FIELD_SKIP_ZERO)) { uint i; if (!memcmp((byte*) start_pos,zero_string,count->field_length)) @@ -947,7 +947,7 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, new_length=calc_packed_length(huff_counts,0); if (old_length < new_length && huff_counts->field_length > 1) { - huff_counts->field_type=FIELD_SKIPP_ZERO; + huff_counts->field_type=FIELD_SKIP_ZERO; huff_counts->counts[0]-=length; huff_counts->bytes_packed=old_length- records/8; goto found_pack; @@ -991,7 +991,7 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->counts[' ']+=huff_counts->tot_pre_space; if (test_space_compress(huff_counts,records,huff_counts->max_end_space, huff_counts->end_space, - huff_counts->tot_end_space,FIELD_SKIPP_ENDSPACE)) + huff_counts->tot_end_space,FIELD_SKIP_ENDSPACE)) goto found_pack; huff_counts->counts[' ']-=huff_counts->tot_pre_space; } @@ -999,7 +999,7 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, { if (test_space_compress(huff_counts,records,huff_counts->max_pre_space, huff_counts->pre_space, - huff_counts->tot_pre_space,FIELD_SKIPP_PRESPACE)) + huff_counts->tot_pre_space,FIELD_SKIP_PRESPACE)) goto found_pack; } @@ -1009,10 +1009,10 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, if (huff_counts->max_zero_fill && (huff_counts->field_type == FIELD_NORMAL || - huff_counts->field_type == FIELD_SKIPP_ZERO)) + huff_counts->field_type == FIELD_SKIP_ZERO)) { huff_counts->counts[0]-=huff_counts->max_zero_fill* - (huff_counts->field_type == FIELD_SKIPP_ZERO ? + (huff_counts->field_type == FIELD_SKIP_ZERO ? records - huff_counts->zero_fields : records); huff_counts->pack_type|=PACK_TYPE_ZERO_FILL; huff_counts->bytes_packed=calc_packed_length(huff_counts,0); @@ -1052,9 +1052,9 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, if (verbose) printf("\nnormal: %3d empty-space: %3d empty-zero: %3d empty-fill: %3d\npre-space: %3d end-space: %3d intervall-fields: %3d zero: %3d\n", field_count[FIELD_NORMAL],space_fields, - field_count[FIELD_SKIPP_ZERO],fill_zero_fields, - field_count[FIELD_SKIPP_PRESPACE], - field_count[FIELD_SKIPP_ENDSPACE], + field_count[FIELD_SKIP_ZERO],fill_zero_fields, + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], field_count[FIELD_INTERVALL], field_count[FIELD_ZERO]); DBUG_VOID_RETURN; @@ -1282,7 +1282,8 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) return 0; } -static int compare_tree(register const uchar *s, register const uchar *t) +static int compare_tree(void* cmp_arg __attribute__((unused)), + register const uchar *s, register const uchar *t) { uint length; for (length=global_count->field_length; length-- ;) @@ -1719,7 +1720,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) field_length-=count->max_zero_fill; switch(count->field_type) { - case FIELD_SKIPP_ZERO: + case FIELD_SKIP_ZERO: if (!memcmp((byte*) start_pos,zero_string,field_length)) { write_bits(1,1); @@ -1733,7 +1734,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); break; - case FIELD_SKIPP_ENDSPACE: + case FIELD_SKIP_ENDSPACE: for (pos=end_pos ; pos > start_pos && pos[-1] == ' ' ; pos--) ; length=(uint) (end_pos-pos); if (count->pack_type & PACK_TYPE_SELECTED) @@ -1756,7 +1757,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) (uint) tree->code_len[(uchar) *start_pos]); start_pos=end_pos; break; - case FIELD_SKIPP_PRESPACE: + case FIELD_SKIP_PRESPACE: for (pos=start_pos ; pos < end_pos && pos[0] == ' ' ; pos++) ; length=(uint) (pos-start_pos); if (count->pack_type & PACK_TYPE_SELECTED) @@ -2045,7 +2046,7 @@ static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length, share->changed=1; /* Force write of header */ share->state.open_count=0; share->global_changed=0; - VOID(my_chsize(share->kfile,share->state.state.key_file_length, + VOID(my_chsize(share->kfile, share->state.state.key_file_length, 0, MYF(0))); if (share->base.keys) isamchk_neaded=1; @@ -2087,7 +2088,7 @@ static void mrg_reset(PACK_MRG_INFO *mrg) { if (mrg->current) { - mi_extra(*mrg->current,HA_EXTRA_NO_CACHE); + mi_extra(*mrg->current, HA_EXTRA_NO_CACHE, 0); mrg->current=0; } } @@ -2102,8 +2103,8 @@ static int mrg_rrnd(PACK_MRG_INFO *info,byte *buf) { isam_info= *(info->current=info->file); info->end=info->current+info->count; - mi_extra(isam_info,HA_EXTRA_RESET); - mi_extra(isam_info,HA_EXTRA_CACHE); + mi_extra(isam_info, HA_EXTRA_RESET, 0); + mi_extra(isam_info, HA_EXTRA_CACHE, 0); filepos=isam_info->s->pack.header_length; } else @@ -2119,14 +2120,14 @@ static int mrg_rrnd(PACK_MRG_INFO *info,byte *buf) filepos, 1)) || error != HA_ERR_END_OF_FILE) return (error); - mi_extra(isam_info,HA_EXTRA_NO_CACHE); + mi_extra(isam_info,HA_EXTRA_NO_CACHE, 0); if (info->current+1 == info->end) return(HA_ERR_END_OF_FILE); info->current++; isam_info= *info->current; filepos=isam_info->s->pack.header_length; - mi_extra(isam_info,HA_EXTRA_RESET); - mi_extra(isam_info,HA_EXTRA_CACHE); + mi_extra(isam_info,HA_EXTRA_RESET, 0); + mi_extra(isam_info,HA_EXTRA_CACHE, 0); } } diff --git a/myisam/sort.c b/myisam/sort.c index 161a5f92bf5..f45ecbaf3a1 100644 --- a/myisam/sort.c +++ b/myisam/sort.c @@ -1,15 +1,15 @@ /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ @@ -19,7 +19,7 @@ them in sorted order through SORT_INFO functions. */ -#include "myisamdef.h" +#include "fulltext.h" #if defined(MSDOS) || defined(__WIN__) #include <fcntl.h> #else @@ -27,7 +27,8 @@ #endif #include <queues.h> - /* static variabels */ +/* static variables */ + #undef MIN_SORT_MEMORY #undef MYF_RW #undef DISK_BUFFER_SIZE @@ -35,64 +36,80 @@ #define MERGEBUFF 15 #define MERGEBUFF2 31 #define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD) -#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) +#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) #define DISK_BUFFER_SIZE (IO_SIZE*16) typedef struct st_buffpek { - my_off_t file_pos; /* Where we are in the sort file */ - uchar *base,*key; /* Key pointers */ - ha_rows count; /* Number of rows in table */ - ulong mem_count; /* numbers of keys in memory */ - ulong max_keys; /* Max keys in buffert */ + my_off_t file_pos; /* Where we are in the sort file */ + uchar *base,*key; /* Key pointers */ + ha_rows count; /* Number of rows in table */ + ulong mem_count; /* numbers of keys in memory */ + ulong max_keys; /* Max keys in buffert */ } BUFFPEK; extern void print_error _VARARGS((const char *fmt,...)); - /* functions defined in this file */ +/* Functions defined in this file */ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info,uint keys, - uchar **sort_keys, - BUFFPEK *buffpek,int *maxbuffer, - IO_CACHE *tempfile); + uchar **sort_keys, + DYNAMIC_ARRAY *buffpek,int *maxbuffer, + IO_CACHE *tempfile, + IO_CACHE *tempfile_for_exceptions); static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar * *sort_keys, - uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); + uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); +static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, + IO_CACHE *tempfile); static int NEAR_F write_index(MI_SORT_PARAM *info,uchar * *sort_keys, - uint count); + uint count); static int NEAR_F merge_many_buff(MI_SORT_PARAM *info,uint keys, - uchar * *sort_keys, - BUFFPEK *buffpek,int *maxbuffer, - IO_CACHE *t_file); + uchar * *sort_keys, + BUFFPEK *buffpek,int *maxbuffer, + IO_CACHE *t_file); static uint NEAR_F read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, - uint sort_length); + uint sort_length); static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys, - IO_CACHE *from_file, IO_CACHE *to_file, - uchar * *sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb); + IO_CACHE *from_file, IO_CACHE *to_file, + uchar * *sort_keys, BUFFPEK *lastbuff, + BUFFPEK *Fb, BUFFPEK *Tb); static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, - IO_CACHE *); -static char **make_char_array(uint fields,uint length,myf my_flag); + IO_CACHE *); + - /* Creates a index of sorted keys */ - /* Returns 0 if everything went ok */ +/* + Creates a index of sorted keys + + SYNOPSIS + _create_index_by_sort() + info Sort parameters + no_messages Set to 1 if no output + sortbuff_size Size if sortbuffer to allocate + + RESULT + 0 ok + <> 0 Error +*/ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong sortbuff_size) { int error,maxbuffer,skr; uint memavl,old_memavl,keys,sort_length; - BUFFPEK *buffpek; + DYNAMIC_ARRAY buffpek; ha_rows records; uchar **sort_keys; - IO_CACHE tempfile; + IO_CACHE tempfile, tempfile_for_exceptions; DBUG_ENTER("_create_index_by_sort"); DBUG_PRINT("enter",("sort_length: %d", info->key_length)); my_b_clear(&tempfile); - buffpek= (BUFFPEK *) NULL; sort_keys= (uchar **) NULL; error= 1; + my_b_clear(&tempfile_for_exceptions); + bzero((char*) &buffpek,sizeof(buffpek)); + sort_keys= (uchar **) NULL; error= 1; maxbuffer=1; memavl=max(sortbuff_size,MIN_SORT_MEMORY); - records= info->max_records; + records= info->sort_info->max_records; sort_length= info->key_length; LINT_INIT(keys); @@ -116,14 +133,14 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, } while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); - if ((sort_keys= (uchar **) make_char_array(keys,sort_length,MYF(0)))) + if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+ + HA_FT_MAXLEN, MYF(0)))) { - if ((buffpek = (BUFFPEK*) my_malloc((uint) (sizeof(BUFFPEK)* - (uint) maxbuffer), - MYF(0)))) - break; - else + if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer, + maxbuffer/2)) my_free((gptr) sort_keys,MYF(0)); + else + break; } old_memavl=memavl; if ((memavl=memavl/4*3) < MIN_SORT_MEMORY && old_memavl > MIN_SORT_MEMORY) @@ -139,7 +156,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (!no_messages) printf(" - Searching for keys, allocating buffer for %d keys\n",keys); - if ((records=find_all_keys(info,keys,sort_keys,buffpek,&maxbuffer,&tempfile)) + if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, + &tempfile,&tempfile_for_exceptions)) == HA_POS_ERROR) goto err; /* purecov: tested */ if (maxbuffer == 0) @@ -156,111 +174,444 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, { if (!no_messages) printf(" - Merging %lu keys\n",records); /* purecov: tested */ - if (merge_many_buff(info,keys,sort_keys,buffpek,&maxbuffer,&tempfile)) - goto err; /* purecov: inspected */ + if (merge_many_buff(info,keys,sort_keys, + dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) + goto err; /* purecov: inspected */ } if (flush_io_cache(&tempfile) || reinit_io_cache(&tempfile,READ_CACHE,0L,0,0)) - goto err; /* purecov: inspected */ + goto err; /* purecov: inspected */ if (!no_messages) - puts(" - Last merge and dumping keys"); /* purecov: tested */ - if (merge_index(info,keys,sort_keys,buffpek,maxbuffer,&tempfile)) - goto err; /* purecov: inspected */ + puts(" - Last merge and dumping keys\n"); /* purecov: tested */ + if (merge_index(info,keys,sort_keys,dynamic_element(&buffpek,0,BUFFPEK *), + maxbuffer,&tempfile)) + goto err; /* purecov: inspected */ } + + if (flush_pending_blocks(info)) + goto err; + + if (my_b_inited(&tempfile_for_exceptions)) + { + MI_INFO *index=info->sort_info->info; + uint keyno=info->key; + uint key_length, ref_length=index->s->rec_reflength; + + if (flush_io_cache(&tempfile_for_exceptions) || + reinit_io_cache(&tempfile_for_exceptions,READ_CACHE,0L,0,0)) + goto err; + + while (!my_b_read(&tempfile_for_exceptions,(byte*)&key_length, + sizeof(key_length)) + && !my_b_read(&tempfile_for_exceptions,(byte*)sort_keys, + (uint) key_length)) + { + if (_mi_ck_write(index,keyno,(uchar*) sort_keys,key_length-ref_length)) + goto err; + } + } + error =0; err: if (sort_keys) my_free((gptr) sort_keys,MYF(0)); - if (buffpek) - my_free((gptr) buffpek,MYF(0)); + delete_dynamic(&buffpek); close_cached_file(&tempfile); + close_cached_file(&tempfile_for_exceptions); DBUG_RETURN(error ? -1 : 0); } /* _create_index_by_sort */ - /* Search after all keys and place them in a temp. file */ +/* Search after all keys and place them in a temp. file */ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, - uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *tempfile) + uchar **sort_keys, DYNAMIC_ARRAY *buffpek, + int *maxbuffer, IO_CACHE *tempfile, + IO_CACHE *tempfile_for_exceptions) { int error; - uint idx,indexpos; + uint idx; DBUG_ENTER("find_all_keys"); - idx=indexpos=error=0; + idx=error=0; + sort_keys[0]=(uchar*) (sort_keys+keys); - while (!(error=(*info->key_read)(info->sort_info,sort_keys[idx]))) + while (!(error=(*info->key_read)(info,sort_keys[idx]))) { - if ((uint) ++idx == keys) + if (info->real_key_length > info->key_length) { - if (indexpos >= (uint) *maxbuffer || - write_keys(info,sort_keys,idx-1,buffpek+indexpos,tempfile)) - DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + if (write_key(info,sort_keys[idx],tempfile_for_exceptions)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + continue; + } + + if (++idx == keys) + { + if (write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), + tempfile)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + + sort_keys[0]=(uchar*) (sort_keys+keys); memcpy(sort_keys[0],sort_keys[idx-1],(size_t) info->key_length); - idx=1; indexpos++; + idx=1; } + sort_keys[idx]=sort_keys[idx-1]+info->key_length; } if (error > 0) DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ /* purecov: inspected */ - if (indexpos) - if (indexpos >= (uint) *maxbuffer || - write_keys(info,sort_keys,idx,buffpek+indexpos,tempfile)) - DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ - *maxbuffer=(int) indexpos; - DBUG_RETURN(indexpos*(keys-1)+idx); + if (buffpek->elements) + { + if (write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), + tempfile)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + *maxbuffer=buffpek->elements-1; + } + else + *maxbuffer=0; + + DBUG_RETURN((*maxbuffer)*(keys-1)+idx); } /* find_all_keys */ - /* Write all keys in memory to file for later merge */ +/* Search after all keys and place them in a temp. file */ + +pthread_handler_decl(thr_find_all_keys,arg) +{ + MI_SORT_PARAM *info= (MI_SORT_PARAM*) arg; + int error; + uint memavl,old_memavl,keys,sort_length; + uint idx, maxbuffer; + uchar **sort_keys=0; + + error=1; + + if (my_thread_init()) + goto err; + if (info->sort_info->got_error) + goto err; + + my_b_clear(&info->tempfile); + my_b_clear(&info->tempfile_for_exceptions); + bzero((char*) &info->buffpek,sizeof(info->buffpek)); + bzero((char*) &info->unique, sizeof(info->unique)); + sort_keys= (uchar **) NULL; + + memavl=max(info->sortbuff_size, MIN_SORT_MEMORY); + idx= info->sort_info->max_records; + sort_length= info->key_length; + maxbuffer= 1; + + while (memavl >= MIN_SORT_MEMORY) + { + if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= + (my_off_t) memavl) + keys= idx+1; + else + { + uint skr; + do + { + skr=maxbuffer; + if (memavl < sizeof(BUFFPEK)*maxbuffer || + (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ + (sort_length+sizeof(char*))) <= 1) + { + mi_check_print_error(info->sort_info->param, + "sort_buffer_size is to small"); + goto err; + } + } + while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr); + } + if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+ + ((info->keyinfo->flag & HA_FULLTEXT) ? + HA_FT_MAXLEN : 0), MYF(0)))) + { + if (my_init_dynamic_array(&info->buffpek, sizeof(BUFFPEK), + maxbuffer, maxbuffer/2)) + my_free((gptr) sort_keys,MYF(0)); + else + break; + } + old_memavl=memavl; + if ((memavl=memavl/4*3) < MIN_SORT_MEMORY && old_memavl > MIN_SORT_MEMORY) + memavl=MIN_SORT_MEMORY; + } + if (memavl < MIN_SORT_MEMORY) + { + mi_check_print_error(info->sort_info->param,"Sort buffer to small"); /* purecov: tested */ + goto err; /* purecov: tested */ + } + + if (info->sort_info->param->testflag & T_VERBOSE) + printf("Key %d - Allocating buffer for %d keys\n",info->key+1,keys); + info->sort_keys=sort_keys; + + idx=error=0; + sort_keys[0]=(uchar*) (sort_keys+keys); + + while (!(error=info->sort_info->got_error) && + !(error=(*info->key_read)(info,sort_keys[idx]))) + { + if (info->real_key_length > info->key_length) + { + if (write_key(info,sort_keys[idx], &info->tempfile_for_exceptions)) + goto err; + continue; + } + + if (++idx == keys) + { + if (write_keys(info,sort_keys,idx-1, + (BUFFPEK *)alloc_dynamic(&info->buffpek), + &info->tempfile)) + goto err; + sort_keys[0]=(uchar*) (sort_keys+keys); + memcpy(sort_keys[0],sort_keys[idx-1],(size_t) info->key_length); + idx=1; + } + sort_keys[idx]=sort_keys[idx-1]+info->key_length; + } + if (error > 0) + goto err; + if (info->buffpek.elements) + { + if (write_keys(info,sort_keys, idx, + (BUFFPEK *) alloc_dynamic(&info->buffpek), &info->tempfile)) + goto err; + info->keys=(info->buffpek.elements-1)*(keys-1)+idx; + } + else + info->keys=idx; + + info->sort_keys_length=keys; + goto ok; + +err: + info->sort_info->got_error=1; /* no need to protect this with a mutex */ + if (sort_keys) + my_free((gptr) sort_keys,MYF(0)); + info->sort_keys=0; + delete_dynamic(& info->buffpek); + close_cached_file(&info->tempfile); + close_cached_file(&info->tempfile_for_exceptions); + +ok: + remove_io_thread(&info->read_cache); + pthread_mutex_lock(&info->sort_info->mutex); + info->sort_info->threads_running--; + pthread_cond_signal(&info->sort_info->cond); + pthread_mutex_unlock(&info->sort_info->mutex); + my_thread_end(); + return NULL; +} + + +int thr_write_keys(MI_SORT_PARAM *sort_param) +{ + SORT_INFO *sort_info=sort_param->sort_info; + MI_CHECK *param=sort_info->param; + ulong length, keys; + ulong *rec_per_key_part=param->rec_per_key_part; + int got_error=sort_info->got_error; + uint i; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; + MI_SORT_PARAM *sinfo; + byte *mergebuf=0; + LINT_INIT(length); + + for (i= 0, sinfo= sort_param ; + i < sort_info->total_keys ; + i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++) + { + if (!sinfo->sort_keys) + { + got_error=1; + continue; + } + if (!got_error) + { + share->state.key_map|=(ulonglong) 1 << sinfo->key; + if (param->testflag & T_STATISTICS) + update_key_parts(sinfo->keyinfo, rec_per_key_part, + sinfo->unique, (ulonglong) info->state->records); + if (!sinfo->buffpek.elements) + { + if (param->testflag & T_VERBOSE) + { + printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys); + fflush(stdout); + } + if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || + flush_pending_blocks(sinfo)) + got_error=1; + } + } + my_free((gptr) sinfo->sort_keys,MYF(0)); + my_free(mi_get_rec_buff_ptr(info, sinfo->rec_buff), + MYF(MY_ALLOW_ZERO_PTR)); + sinfo->sort_keys=0; + } + + for (i= 0, sinfo= sort_param ; + i < sort_info->total_keys ; + i++, + delete_dynamic(&sinfo->buffpek), + close_cached_file(&sinfo->tempfile), + close_cached_file(&sinfo->tempfile_for_exceptions), + sinfo++) + { + if (got_error) + continue; + if (sinfo->buffpek.elements) + { + uint maxbuffer=sinfo->buffpek.elements-1; + if (!mergebuf) + { + length=param->sort_buffer_length; + while (length >= MIN_SORT_MEMORY && !mergebuf) + { + mergebuf=my_malloc(length, MYF(0)); + length=length*3/4; + } + if (!mergebuf) + { + got_error=1; + continue; + } + } + keys=length/sinfo->key_length; + if (maxbuffer >= MERGEBUFF2) + { + if (param->testflag & T_VERBOSE) + printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); + if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, + dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), + (int*) &maxbuffer, &sinfo->tempfile)) + { + got_error=1; + continue; + } + } + if (flush_io_cache(&sinfo->tempfile) || + reinit_io_cache(&sinfo->tempfile,READ_CACHE,0L,0,0)) + { + got_error=1; + continue; + } + if (param->testflag & T_VERBOSE) + printf("Key %d - Last merge and dumping keys\n", sinfo->key+1); + if (merge_index(sinfo, keys, (uchar **)mergebuf, + dynamic_element(&sinfo->buffpek,0,BUFFPEK *), + maxbuffer,&sinfo->tempfile) || + flush_pending_blocks(sinfo)) + { + got_error=1; + continue; + } + } + if (my_b_inited(&sinfo->tempfile_for_exceptions)) + { + uint key_length; + + if (param->testflag & T_VERBOSE) + printf("Key %d - Dumping 'long' keys\n", sinfo->key+1); + + if (flush_io_cache(&sinfo->tempfile_for_exceptions) || + reinit_io_cache(&sinfo->tempfile_for_exceptions,READ_CACHE,0L,0,0)) + { + got_error=1; + continue; + } + + while (!got_error && + !my_b_read(&sinfo->tempfile_for_exceptions,(byte*)&key_length, + sizeof(key_length)) && + !my_b_read(&sinfo->tempfile_for_exceptions,(byte*)mergebuf, + (uint) key_length)) + { + if (_mi_ck_write(info,sinfo->key,(uchar*) mergebuf, + key_length - info->s->rec_reflength)) + got_error=1; + } + } + } + my_free((gptr) mergebuf,MYF(MY_ALLOW_ZERO_PTR)); + return got_error; +} + + /* Write all keys in memory to file for later merge */ static int NEAR_F write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, - uint count, BUFFPEK *buffpek, - IO_CACHE *tempfile) + uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) { uchar **end; uint sort_length=info->key_length; DBUG_ENTER("write_keys"); qsort2((byte*) sort_keys,count,sizeof(byte*),(qsort2_cmp) info->key_cmp, - info->sort_info); + info); if (!my_b_inited(tempfile) && open_cached_file(tempfile, info->tmpdir, "ST", DISK_BUFFER_SIZE, - info->myf_rw)) + info->sort_info->param->myf_rw)) DBUG_RETURN(1); /* purecov: inspected */ + buffpek->file_pos=my_b_tell(tempfile); buffpek->count=count; for (end=sort_keys+count ; sort_keys != end ; sort_keys++) + { if (my_b_write(tempfile,(byte*) *sort_keys,(uint) sort_length)) DBUG_RETURN(1); /* purecov: inspected */ + } DBUG_RETURN(0); } /* write_keys */ - /* Write index */ +static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, + IO_CACHE *tempfile) +{ + uint key_length=info->real_key_length; + DBUG_ENTER("write_key"); + + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, info->tmpdir, "ST", DISK_BUFFER_SIZE, + info->sort_info->param->myf_rw)) + DBUG_RETURN(1); + + if (my_b_write(tempfile,(byte*)&key_length,sizeof(key_length)) || + my_b_write(tempfile,(byte*)key,(uint) key_length)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} /* write_key */ + + +/* Write index */ static int NEAR_F write_index(MI_SORT_PARAM *info, register uchar **sort_keys, - register uint count) + register uint count) { DBUG_ENTER("write_index"); qsort2((gptr) sort_keys,(size_t) count,sizeof(byte*), - (qsort2_cmp) info->key_cmp,info->sort_info); + (qsort2_cmp) info->key_cmp,info); while (count--) - if ((*info->key_write)(info->sort_info,*sort_keys++)) + { + if ((*info->key_write)(info,*sort_keys++)) DBUG_RETURN(-1); /* purecov: inspected */ + } DBUG_RETURN(0); } /* write_index */ - /* Merge buffers to make < MERGEBUFF2 buffers */ + /* Merge buffers to make < MERGEBUFF2 buffers */ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, - uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *t_file) + uchar **sort_keys, BUFFPEK *buffpek, + int *maxbuffer, IO_CACHE *t_file) { register int i; IO_CACHE t_file2, *from_file, *to_file, *temp; @@ -268,11 +619,11 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, DBUG_ENTER("merge_many_buff"); if (*maxbuffer < MERGEBUFF2) - DBUG_RETURN(0); /* purecov: inspected */ + DBUG_RETURN(0); /* purecov: inspected */ if (flush_io_cache(t_file) || open_cached_file(&t_file2,info->tmpdir,"ST",DISK_BUFFER_SIZE, - info->myf_rw)) - DBUG_RETURN(1); /* purecov: inspected */ + info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ from_file= t_file ; to_file= &t_file2; while (*maxbuffer >= MERGEBUFF2) @@ -283,30 +634,40 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) { if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+i+MERGEBUFF-1)) - break; /* purecov: inspected */ + buffpek+i,buffpek+i+MERGEBUFF-1)) + break; /* purecov: inspected */ } if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+ *maxbuffer)) + buffpek+i,buffpek+ *maxbuffer)) break; /* purecov: inspected */ if (flush_io_cache(to_file)) - break; /* purecov: inspected */ + break; /* purecov: inspected */ temp=from_file; from_file=to_file; to_file=temp; *maxbuffer= (int) (lastbuff-buffpek)-1; } - close_cached_file(to_file); /* This holds old result */ + close_cached_file(to_file); /* This holds old result */ if (to_file == t_file) - *t_file=t_file2; /* Copy result file */ + *t_file=t_file2; /* Copy result file */ - DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ + DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ } /* merge_many_buff */ - /* Read data to buffer */ - /* This returns (uint) -1 if something goes wrong */ +/* + Read data to buffer + + SYNOPSIS + read_to_buffer() + fromfile File to read from + buffpek Where to read from + sort_length max length to read + RESULT + > 0 Ammount of bytes read + -1 Error +*/ static uint NEAR_F read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, - uint sort_length) + uint sort_length) { register uint count; uint length; @@ -314,24 +675,26 @@ static uint NEAR_F read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) { if (my_pread(fromfile->file,(byte*) buffpek->base, - (length= sort_length*count),buffpek->file_pos,MYF_RW)) - return((uint) -1); /* purecov: inspected */ + (length= sort_length*count),buffpek->file_pos,MYF_RW)) + return((uint) -1); /* purecov: inspected */ buffpek->key=buffpek->base; - buffpek->file_pos+= length; /* New filepos */ - buffpek->count-= count; + buffpek->file_pos+= length; /* New filepos */ + buffpek->count-= count; buffpek->mem_count= count; } return (count*sort_length); } /* read_to_buffer */ - /* Merge buffers to one buffer */ - /* If to_file == 0 then use info->key_write */ +/* + Merge buffers to one buffer + If to_file == 0 then use info->key_write +*/ static int NEAR_F -merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, - IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb) +merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, + IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, + BUFFPEK *Fb, BUFFPEK *Tb) { int error; uint sort_length,maxcount; @@ -351,8 +714,8 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, sort_length=info->key_length; if (init_queue(&queue,(uint) (Tb-Fb)+1,offsetof(BUFFPEK,key),0, - (int (*)(void*, byte *,byte*)) info->key_cmp, - (void*) info->sort_info)) + (int (*)(void*, byte *,byte*)) info->key_cmp, + (void*) info)) DBUG_RETURN(1); /* purecov: inspected */ for (buffpek= Fb ; buffpek <= Tb ; buffpek++) @@ -361,7 +724,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek->base= strpos; buffpek->max_keys=maxcount; strpos+= (uint) (error=(int) read_to_buffer(from_file,buffpek, - sort_length)); + sort_length)); if (error == -1) goto err; /* purecov: inspected */ queue_insert(&queue,(char*) buffpek); @@ -374,52 +737,52 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek=(BUFFPEK*) queue_top(&queue); if (to_file) { - if (my_b_write(to_file,(byte*) buffpek->key,(uint) sort_length)) - { - error=1; goto err; /* purecov: inspected */ - } + if (my_b_write(to_file,(byte*) buffpek->key,(uint) sort_length)) + { + error=1; goto err; /* purecov: inspected */ + } } else { - if ((*info->key_write)(info->sort_info,(void*) buffpek->key)) - { - error=1; goto err; /* purecov: inspected */ - } + if ((*info->key_write)(info,(void*) buffpek->key)) + { + error=1; goto err; /* purecov: inspected */ + } } buffpek->key+=sort_length; if (! --buffpek->mem_count) { - if (!(error=(int) read_to_buffer(from_file,buffpek,sort_length))) - { - uchar *base=buffpek->base; - uint max_keys=buffpek->max_keys; - - VOID(queue_remove(&queue,0)); - - /* Put room used by buffer to use in other buffer */ - for (refpek= (BUFFPEK**) &queue_top(&queue); - refpek <= (BUFFPEK**) &queue_end(&queue); - refpek++) - { - buffpek= *refpek; - if (buffpek->base+buffpek->max_keys*sort_length == base) - { - buffpek->max_keys+=max_keys; - break; - } - else if (base+max_keys*sort_length == buffpek->base) - { - buffpek->base=base; - buffpek->max_keys+=max_keys; - break; - } - } - break; /* One buffer have been removed */ - } + if (!(error=(int) read_to_buffer(from_file,buffpek,sort_length))) + { + uchar *base=buffpek->base; + uint max_keys=buffpek->max_keys; + + VOID(queue_remove(&queue,0)); + + /* Put room used by buffer to use in other buffer */ + for (refpek= (BUFFPEK**) &queue_top(&queue); + refpek <= (BUFFPEK**) &queue_end(&queue); + refpek++) + { + buffpek= *refpek; + if (buffpek->base+buffpek->max_keys*sort_length == base) + { + buffpek->max_keys+=max_keys; + break; + } + else if (base+max_keys*sort_length == buffpek->base) + { + buffpek->base=base; + buffpek->max_keys+=max_keys; + break; + } + } + break; /* One buffer have been removed */ + } } else if (error == -1) - goto err; /* purecov: inspected */ - queue_replaced(&queue); /* Top element has been replaced */ + goto err; /* purecov: inspected */ + queue_replaced(&queue); /* Top element has been replaced */ } } buffpek=(BUFFPEK*) queue_top(&queue); @@ -430,9 +793,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, if (to_file) { if (my_b_write(to_file,(byte*) buffpek->key, - (sort_length*buffpek->mem_count))) + (sort_length*buffpek->mem_count))) { - error=1; goto err; /* purecov: inspected */ + error=1; goto err; /* purecov: inspected */ } } else @@ -440,18 +803,18 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, register uchar *end; strpos= buffpek->key; for (end=strpos+buffpek->mem_count*sort_length; - strpos != end ; - strpos+=sort_length) + strpos != end ; + strpos+=sort_length) { - if ((*info->key_write)(info->sort_info,(void*) strpos)) - { - error=1; goto err; /* purecov: inspected */ - } + if ((*info->key_write)(info,(void*) strpos)) + { + error=1; goto err; /* purecov: inspected */ + } } } } while ((error=(int) read_to_buffer(from_file,buffpek,sort_length)) != -1 && - error != 0); + error != 0); lastbuff->count=count; if (to_file) @@ -462,34 +825,16 @@ err: } /* merge_buffers */ - /* Do a merge to output-file (save only positions) */ + /* Do a merge to output-file (save only positions) */ static int NEAR_F merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, - BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) + BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) { DBUG_ENTER("merge_index"); if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, - buffpek+maxbuffer)) + buffpek+maxbuffer)) DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(0); } /* merge_index */ - - /* Make a pointer of arrays to keys */ - -static char **make_char_array(register uint fields, uint length, myf my_flag) -{ - register char **pos; - char **old_pos,*char_pos; - DBUG_ENTER("make_char_array"); - - if ((old_pos= (char**) my_malloc( fields*(length+sizeof(char*)), my_flag))) - { - pos=old_pos; char_pos=((char*) (pos+fields)) -length; - while (fields--) - *(pos++) = (char_pos+= length); - } - - DBUG_RETURN(old_pos); -} /* make_char_array */ |