diff options
Diffstat (limited to 'myisam')
68 files changed, 25086 insertions, 0 deletions
diff --git a/myisam/.cvsignore b/myisam/.cvsignore new file mode 100644 index 00000000000..d9adcedd308 --- /dev/null +++ b/myisam/.cvsignore @@ -0,0 +1,12 @@ +.deps +.libs +Makefile +Makefile.in +ft_eval +ft_test1 +mi_test1 +mi_test2 +mi_test3 +myisamchk +myisamlog +myisampack diff --git a/myisam/Attic/ft_global.h b/myisam/Attic/ft_global.h new file mode 100644 index 00000000000..c8d0734fed3 --- /dev/null +++ b/myisam/Attic/ft_global.h @@ -0,0 +1,52 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* some definitions for full-text indices */ + +/* #include "myisam.h" */ + +#ifndef _ft_global_h +#define _ft_global_h +#ifdef __cplusplus +extern "C" { +#endif + +#define FT_QUERY_MAXLEN 1024 + +typedef struct ft_doc_rec { + my_off_t dpos; + double weight; +} FT_DOC; + +typedef struct st_ft_doclist { + int ndocs; + int curdoc; + void *info; /* actually (MI_INFO *) but don't want to include myisam.h */ + FT_DOC doc[1]; +} FT_DOCLIST; + +int ft_init_stopwords(const char **); + +FT_DOCLIST * ft_init_search(MI_INFO *, uint, byte *, uint, bool); +double ft_read_next(FT_DOCLIST *, char *); +#define ft_close_search(handler) my_free(((gptr)(handler)),MYF(0)) + +#ifdef __cplusplus +} +#endif +#endif diff --git a/myisam/ChangeLog b/myisam/ChangeLog new file mode 100644 index 00000000000..a2cd00a8c50 --- /dev/null +++ b/myisam/ChangeLog @@ -0,0 +1,137 @@ +2000-07-02 Michael Widenius <monty@mysql.com> + +* Added safety margin to guard against full index file. + +2000-05-22 Michael Widenius <monty@mysql.com> + +* Fixed that --join works with myisampack. + +2000-05-14 Michael Widenius <monty@mysql.com> + +* Don't lock datafile during myisamchk (only indexfile is locked; This is good + enough for all MyISAM functions); This made it possible to close datafile + in rep_by_sort(). + +2000-05-04 Michael Widenius <monty@mysql.com> + +* Fixed bug in code that scanned after rows in a crashed table. + This could cause an infinite loop when repairing tables. + +2000-04-26 Michael Widenius <monty@mysql.com> + +* Fixed bug when doing read_next after a delete/insert which balanced key + pages (In this case one internal buffer was wrongly reused) + +2000-04-21 Michael Widenius <monty@tik.pp.sci.fi> + +* Changed mi_find_halfpos() to return key, key_length and pos after key. +* Don't join or split key buffers in the middle when inserting a key + that is bigger than all other keys; This will improve inserts when + doing these in sorted order. + +2000-04-04 Michael Widenius <monty@mysql.com> + +* Added support for different languages on key part level. + +2000-02-23 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed that myisamchk works properly with RAID. + +2000-02-07 Michael Widenius <monty@tik.pp.sci.fi> + +* Added delete and rename of tables (works with RAID tables) + +2000-01-29 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed the sorting of index works with prefix-packed keys. + +1999-11-24 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed that DECIMAL() keys are sorted correct for negative numbers. + +1999-11-22 Michael Widenius <monty@monty.pp.sci.fi> + +* removed 'NO_LOCKING' macros. +* Added function mi_rnext_same +* Added support for concurrent reads. + +1999-11-05 Michael Widenius <monty@tik.pp.sci.fi> + +* Added function mi_scan(). +* Changed all functions to return error number in case of errors. + +1999-08-17 Michael Widenius <monty@tik.pp.sci.fi> + +* Added option DELAY_KEY_WRITE to tables and mi_open() + +1999-08-10 Michael Widenius <monty@tik.pp.sci.fi> + +* Added support of HA_READ_PREFIX_LAST to mi_rkey(). This finds the last + row with the given prefix. + +Mon Aug 2 13:54:35 1999 Michael Widenius <monty@bitch.pp.sci.fi> + +* Added data- and key-file-length to myisamchk. +* Fixed some problems with null and space packed keys. + +1999-07-15 Michael Widenius <monty@tik.pp.sci.fi> + +* The following options are for COUNT(DISTINCT ..) +* Added option HA_EXTRA_NO_ROWS; In this case only the index tree is updated +* Added mi_delete_all_rows() + +1999-07-13 Michael Widenius <monty@tik.pp.sci.fi> + +* Added special handling of tempoary tables + +1999-06-12 Michael Widenius <monty@monty.pp.sci.fi> + +* Added optional checksum for file and for each dynamic-length row +* Added unique constraint checking + +1999-05-06 Michael Widenius <monty@tik.pp.sci.fi> + +* All index blocks of the same size now share the same key block delete link + +1999-03-17 Michael Widenius <monty@monty.pp.sci.fi> + +* Different key packing code depending on if the first key part + is a variable length column (space packed, BLOB or VARCHAR) +* The create interface allows one to specify a key segment to start and + end one a specific bit. (The bit handling isn't yet implemented) +* Added more tests to 'test1' + +1999-03-16 Michael Widenius <monty@monty.pp.sci.fi> + +* Added option -m to myisamchk as an alternative to -e (-m is faster but + not as quite as safe as -e). +* Added option --fast to not check not changed tables. +* The first update will set a bit that the table has been changed. +* The first update to a table increments a 'open_count' field. This will + be reset on close. This will allow myisamchk to find tables that hasn't + been properly closed! +* Support for true VARCHAR columns + +1999-03-01 Michael Widenius <monty@monty.pp.sci.fi> + +* Dynamic length blocks are double linked to allow easy reallocation + of block lengths. This will help that the dynamic length data will not be + as fragmented as with ISAM. +* Extended mypack_isam to compress BLOB/TEXT columns. +* Allow keys on BLOB. + +1999-02-06 Michael Widenius <monty@monty.pp.sci.fi> + +* Keys, key pointers and all varibles in the index file are stored in + high-endian-order to get better compression. +* Allow NULL on keys + +1998-10-29 Michael Widenius <monty@monty.pp.sci.fi> + +* All data is stored in low-endian order + (This means that the files will be architecture and OS independent) +* All record numbers are now of type 'ha_rows' and file pointer are now of type + my_off_t. One can use files with 32-64 bit pointers with a 32bit or 64bit + record handling. + Currently the code is limited to 5 bytes pointers, but this is real easy + to change. diff --git a/myisam/Makefile.am b/myisam/Makefile.am new file mode 100644 index 00000000000..4a26fbae479 --- /dev/null +++ b/myisam/Makefile.am @@ -0,0 +1,107 @@ +# Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +EXTRA_DIST = mi_test_all.sh mi_test_all.res +pkgdata_DATA = mi_test_all mi_test_all.res + +INCLUDES = @MT_INCLUDES@ -I$(srcdir)/../include -I../include +LDADD = @CLIENT_EXTRA_LDFLAGS@ libmyisam.a ../mysys/libmysys.a \ + ../dbug/libdbug.a ../strings/libmystrings.a +pkglib_LIBRARIES = libmyisam.a +bin_PROGRAMS = myisamchk myisamlog myisampack +myisamchk_DEPENDENCIES= $(LIBRARIES) +myisamlog_DEPENDENCIES= $(LIBRARIES) +myisampack_DEPENDENCIES=$(LIBRARIES) +noinst_PROGRAMS = mi_test1 mi_test2 mi_test3 ft_test1 ft_eval +noinst_HEADERS = myisamdef.h fulltext.h ftdefs.h ft_test1.h ft_eval.h +mi_test1_DEPENDENCIES= $(LIBRARIES) +mi_test2_DEPENDENCIES= $(LIBRARIES) +mi_test3_DEPENDENCIES= $(LIBRARIES) +ft_test1_DEPENDENCIES= $(LIBRARIES) +ft_eval_DEPENDENCIES= $(LIBRARIES) +libmyisam_a_SOURCES = mi_open.c mi_extra.c mi_info.c mi_rkey.c \ + mi_rnext.c mi_rnext_same.c \ + mi_search.c mi_page.c mi_key.c mi_locking.c \ + mi_rrnd.c mi_scan.c mi_cache.c \ + mi_statrec.c mi_packrec.c mi_dynrec.c \ + mi_update.c mi_write.c mi_unique.c \ + mi_delete.c \ + mi_rprev.c mi_rfirst.c mi_rlast.c mi_rsame.c \ + mi_rsamepos.c mi_panic.c mi_close.c mi_create.c\ + mi_range.c mi_dbug.c mi_checksum.c mi_log.c \ + mi_changed.c mi_static.c mi_delete_all.c \ + mi_delete_table.c mi_rename.c mi_check.c \ + ft_parser.c ft_search.c ft_stopwords.c ft_static.c \ + ft_update.c sort.c +CLEANFILES = test?.IS? isam.log mi_test_all +DEFS = -DMAP_TO_USE_RAID +# Omit dependency for ../mit-pthreads/include/sys that only exits if +# mit-pthreads are used +OMIT_DEPENDENCIES = pthread.h stdio.h __stdio.h stdlib.h __stdlib.h math.h\ + __math.h time.h __time.h unistd.h __unistd.h types.h \ + xtypes.h ac-types.h posix.h string.h __string.h \ + errno.h socket.h inet.h dirent.h netdb.h \ + cleanup.h cond.h debug_out.h fd.h kernel.h mutex.h \ + prio_queue.h pthread_attr.h pthread_once.h queue.h\ + sleep.h specific.h version.h pwd.h timers.h uio.h \ + cdefs.h machdep.h signal.h __signal.h util.h \ + wait.h + +# Move to automake rules ? +prolint:; plparse -b -u -hF1 "-width(0,0)" "-format=%f:%l:\s%t:%n\s%m" \ + "-elib(????)" "+elib(?3??)" my.lnt $(nisam_SOURCES) + +SUFFIXES = .sh + +.sh: + @RM@ -f $@ $@-t + @SED@ \ + -e 's!@''bindir''@!$(bindir)!g' \ + -e 's!@''scriptdir''@!$(bindir)!g' \ + -e 's!@''prefix''@!$(prefix)!g' \ + -e 's!@''datadir''@!$(datadir)!g' \ + -e 's!@''localstatedir''@!$(localstatedir)!g' \ + -e 's!@''libexecdir''@!$(libexecdir)!g' \ + -e 's!@''CC''@!@CC@!'\ + -e 's!@''CXX''@!@CXX@!'\ + -e 's!@''GXX''@!@GXX@!'\ + -e 's!@''PERL''@!@PERL@!' \ + -e 's!@''CFLAGS''@!@SAVE_CFLAGS@!'\ + -e 's!@''CXXFLAGS''@!@SAVE_CXXFLAGS@!'\ + -e 's!@''LDFLAGS''@!@SAVE_LDFLAGS@!'\ + -e 's!@''VERSION''@!@VERSION@!' \ + -e 's!@''COMPILATION_COMMENT''@!@COMPILATION_COMMENT@!' \ + -e 's!@''MACHINE_TYPE''@!@MACHINE_TYPE@!' \ + -e 's!@''HOSTNAME''@!@HOSTNAME@!' \ + -e 's!@''SYSTEM_TYPE''@!@SYSTEM_TYPE@!' \ + -e 's!@''CHECK_PID''@!@CHECK_PID@!' \ + -e 's!@''FIND_PROC''@!@FIND_PROC@!' \ + -e 's!@''MYSQLD_DEFAULT_SWITCHES''@!@MYSQLD_DEFAULT_SWITCHES@!' \ + -e 's!@''MYSQL_UNIX_ADDR''@!@MYSQL_UNIX_ADDR@!' \ + -e 's!@''IS_LINUX''@!@IS_LINUX@!' \ + -e "s!@""CONF_COMMAND""@!@CONF_COMMAND@!" \ + -e 's!@''MYSQLD_USER''@!@MYSQLD_USER@!' \ + -e 's!@''sysconfdir''@!@sysconfdir@!' \ + -e 's!@''SHORT_MYSQL_INTRO''@!@SHORT_MYSQL_INTRO@!' \ + -e 's!@''SHARED_LIB_VERSION''@!@SHARED_LIB_VERSION@!' \ + -e 's!@''MYSQL_BASE_VERSION''@!@MYSQL_BASE_VERSION@!' \ + -e 's!@''MYSQL_NO_DASH_VERSION''@!@MYSQL_NO_DASH_VERSION@!' \ + -e 's!@''MYSQL_TCP_PORT''@!@MYSQL_TCP_PORT@!' \ + -e 's!@''PERL_DBI_VERSION''@!@PERL_DBI_VERSION@!' \ + -e 's!@''PERL_DBD_VERSION''@!@PERL_DBD_VERSION@!' \ + -e 's!@''PERL_DATA_DUMPER''@!@PERL_DATA_DUMPER@!' \ + $< > $@-t + @MV@ $@-t $@ diff --git a/myisam/NEWS b/myisam/NEWS new file mode 100644 index 00000000000..bb1f141610b --- /dev/null +++ b/myisam/NEWS @@ -0,0 +1,66 @@ +New features compared to NISAM: + +- All file positions have type my_off_t; This enables one to use big + files (2^63 byte) by defining my_off_t to be longlong on OS that supports + big files. +- When creating a table, one can now specify the maximum data file length. + This will be used to calculate the length of row pointers. +- All key segments have their own language definition. +- Some changes to support more types: + The biggest change is that the interface allows MY_ISAM will support + variable length integer types. (Only the interface is implemented) +- All data is stored with low byte first; This makes the data machine + independent. +- All number keys are stored with high byte first to give better packing. +- Support for a true VARCHAR type; A VARCHAR column starts with a length + stored on 2 bytes. +- Tables with VARCHAR may have fixed or dynamic record length. +- There are now 2 different ways to pack keys: + - If the first key part is a space stripped CHAR, a VARCHAR or a BLOB the + 'packed' method is used. This only prefix-compresses the first + key part. + - In other cases prefix packing is used (This also includes the record + pointer into the prefix packing). A key may in the best case be + packed on 2 bytes. +- VARCHAR and CHAR may be up to 65K +- Index on BLOB and VARCHAR. +- One can now have NULL in an index. This takes 0-1 bytes / key. +- MYISAM will allow one to specify one AUTO_INCREMENT column; MYISAM will + automaticly update this on INSERT/UPDATE. The AUTO_INCREMENT value can be + reset with myisamchk. +- Max key length will be 500 by default; In cases of longer keys than 250, + a bigger key block size than the default of 1024 byes is used for this key. +- Max number of keys enlarged to 32 as default. This can be enlarged to 64 + without having to recompile myisamchk. +- There is a flag in the MYISAM header that tells if the index file (.MYI) + was closed correctly. +- myisamchk will now mark tables as checked. 'myisamchk --fast' will only + check those tables that doesn't have this mark. +- 'myisamchk -a' stores statistic for key parts (and not only for whole keys + as in NISAM). +- Dynamic size rows will now be much less fragmented when mixing deletes with + update and insert. This is done by automaticly combine adjacent deleted + blocks and by extending blocks if the next block is deleted. +- For dynamic size rows, the delete link contains a pointer to itself + (to make repairs easier). +- myisampack (called pack_isam in NISAM) can pack BLOB and VARCHAR + columns. +- One can now disable any key from update; In NISAM one could only disable + the last x keys. +- One can have a UNIQUE constraint on anything (including BLOBS). + This is implemented by a key that contains a hashed number of the whole + record and before inserting a new record, MyISAM will check all records + with the same hash for dupplicates. +- When creating the table, one can define that MyISAM should maintain + a CRC for the whole table (to make isamchk even better). In the case of + dynamic size rows, one will in this case get 1 byte checksum for each row. + (This is a great help for debugging, but it can also be used to keep + MyISAM table 'extra' safe. +- Temporary tables will not write not flushed keys to file on close and + not wait on 'disk full' conditions. + +Interface changes compared to NISAM: + +- mi_create() + - keyinfo->seg must be allocated explicitely. + - One must put number of key segments in keyinfo diff --git a/myisam/TODO b/myisam/TODO new file mode 100644 index 00000000000..cad9486e1bb --- /dev/null +++ b/myisam/TODO @@ -0,0 +1,7 @@ +TODO: + +- Let packisam find the optimal way to store keys. +- kill when using 'myisamchk' should remove all temporary files. +- Text search index + (Sergei A. Golub is working on this) +- Add '%' packed to myisamchk for compressed tables with blobs. diff --git a/myisam/ft_eval.c b/myisam/ft_eval.c new file mode 100644 index 00000000000..b8628724642 --- /dev/null +++ b/myisam/ft_eval.c @@ -0,0 +1,208 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" +#include "ft_eval.h" +#include <stdarg.h> +#include <getopt.h> + +static void print_error(int exit_code, const char *fmt,...); + +int main(int argc,char *argv[]) +{ + MI_INFO *file; + int i,j; + + MY_INIT(argv[0]); + get_options(argc,argv); + bzero((char*)recinfo,sizeof(recinfo)); + + /* First define 2 columns */ + recinfo[0].type=FIELD_SKIPP_ENDSPACE; + recinfo[0].length=docid_length; + recinfo[1].type=FIELD_BLOB; + recinfo[1].length= 4+mi_portable_sizeof_char_ptr; + + /* Define a key over the first column */ + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=1; + keyinfo[0].seg[0].type= HA_KEYTYPE_TEXT; + keyinfo[0].seg[0].flag= HA_BLOB_PART; + keyinfo[0].seg[0].start=recinfo[0].length; + keyinfo[0].seg[0].length=key_length; + keyinfo[0].seg[0].null_bit=0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].seg[0].bit_start=4; + keyinfo[0].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[0].flag = HA_FULLTEXT; + + if (!silent) + printf("- Creating isam-file\n"); + if (mi_create(filename,1,keyinfo,2,recinfo,0,NULL,(MI_CREATE_INFO*) 0,0)) + goto err; + if (!(file=mi_open(filename,2,0))) + goto err; + if (!silent) + printf("Initializing stopwords\n"); + ft_init_stopwords(stopwordlist); + + if (!silent) + printf("- Writing key:s\n"); + + my_errno=0; + i=0; + while(create_record(record,df)) + { + error=mi_write(file,record); + if (error) + printf("I= %2d mi_write: %d errno: %d\n",i,error,my_errno); + i++; + } + fclose(df); + + if (mi_close(file)) goto err; + if (!silent) + printf("- Reopening file\n"); + if (!(file=mi_open(filename,2,0))) goto err; + if (!silent) + printf("- Reading rows with key\n"); + for(i=1;create_record(record,qf);i++) { + FT_DOCLIST *result; double w; int t; + + result=ft_init_search(file,0,blob_record,strlen(blob_record),1); + if(!result) { + printf("Query %d failed with errno %3d\n",i,my_errno); + goto err; + } + if (!silent) + printf("Query %d. Found: %d.\n",i,result->ndocs); + for(j=0;(w=ft_read_next(result, read_record))>0;j++) { + t=uint2korr(read_record); + printf("%d %.*s %f\n",i,t,read_record+2,w); + } + if(w<0) { + printf("ft_read_next %d failed with errno %3d\n",j,my_errno); + goto err; + } + ft_close_search(result); + } + + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + + return (0); +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skipp warning */ + +} + +void get_options(int argc,char *argv[]) +{ + int c; + char *options=(char*) "Vh#:qSs:"; + + while ((c=getopt(argc,argv,options)) != -1) + { + switch(c) { + case 's': + if(stopwordlist && stopwordlist!=ft_precompiled_stopwords) break; + { + FILE *f; char s[HA_FT_MAXLEN]; int i=0,n=SWL_INIT; + + if(!(stopwordlist=malloc(n*sizeof(char *)))) + print_error(1,"malloc(%d)",n*sizeof(char *)); + if(!(f=fopen(optarg,"r"))) + print_error(1,"fopen(%s)",optarg); + while(!feof(f)) { + if(!(fgets(s,HA_FT_MAXLEN,f))) + print_error(1,"fgets(s,%d,%s)",HA_FT_MAXLEN,optarg); + if(!(stopwordlist[i++]=strdup(s))) + print_error(1,"strdup(%s)",s); + if(i>=n) { + n+=SWL_PLUS; + if(!(stopwordlist=(const char**) realloc((char*) stopwordlist,n*sizeof(char *)))) + print_error(1,"realloc(%d)",n*sizeof(char *)); + } + } + fclose(f); + stopwordlist[i]=NULL; + break; + } + case 'q': silent=1; break; + case 'S': if(stopwordlist==ft_precompiled_stopwords) stopwordlist=NULL; break; + case '#': + DEBUGGER_ON; + DBUG_PUSH (optarg); + break; + case 'V': + case '?': + case 'h': + default: + printf("%s -[%s] <d_file> <q_file>\n", argv[0], options); + exit(0); + } + } + if(!(d_file=argv[optind])) print_error(1,"No d_file"); + if(!(df=fopen(d_file,"r"))) + print_error(1,"fopen(%s)",d_file); + if(!(q_file=argv[optind+1])) print_error(1,"No q_file"); + if(!(qf=fopen(q_file,"r"))) + print_error(1,"fopen(%s)",q_file); + return; +} /* get options */ + +int create_record(char *pos, FILE *file) +{ uint tmp; char *ptr; + + bzero((char *)pos,MAX_REC_LENGTH); + + /* column 1 - VARCHAR */ + if(!(fgets(pos+2,MAX_REC_LENGTH-32,file))) + { + if(feof(file)) return 0; else print_error(1,"fgets(docid) - 1"); + } + tmp=strlen(pos+2)-1; + int2store(pos,tmp); + pos+=recinfo[0].length; + + /* column 2 - BLOB */ + + if(!(fgets(blob_record,MAX_BLOB_LENGTH,file))) + print_error(1,"fgets(docid) - 2"); + tmp=strlen(blob_record); + int4store(pos,tmp); + ptr=blob_record; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); + return 1; +} + +/* VARARGS */ + +static void print_error(int exit_code, const char *fmt,...) +{ + va_list args; + + va_start(args,fmt); + fprintf(stderr,"%s: error: ",my_progname); + VOID(vfprintf(stderr, fmt, args)); + VOID(fputc('\n',stderr)); + fflush(stderr); + va_end(args); + exit(exit_code); +} diff --git a/myisam/ft_eval.h b/myisam/ft_eval.h new file mode 100644 index 00000000000..5d7f41ab04d --- /dev/null +++ b/myisam/ft_eval.h @@ -0,0 +1,46 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +const char **stopwordlist=ft_precompiled_stopwords; + +#define MAX_REC_LENGTH 128 +#define MAX_BLOB_LENGTH 60000 +char record[MAX_REC_LENGTH], read_record[MAX_REC_LENGTH+MAX_BLOB_LENGTH]; +char blob_record[MAX_BLOB_LENGTH+20*20]; + +char *filename= (char*) "EVAL"; + +int silent=0, error=0; + +uint key_length=MAX_BLOB_LENGTH,docid_length=32; +char *d_file, *q_file; +FILE *df,*qf; + +MI_COLUMNDEF recinfo[3]; +MI_KEYDEF keyinfo[2]; +MI_KEYSEG keyseg[10]; + +void get_options(int argc,char *argv[]); +int create_record(char *, FILE *); + +#define SWL_INIT 500 +#define SWL_PLUS 50 + +#define MAX_LINE_LENGTH 128 +char line[MAX_LINE_LENGTH]; + diff --git a/myisam/ft_parser.c b/myisam/ft_parser.c new file mode 100644 index 00000000000..e2fcd2b00a1 --- /dev/null +++ b/myisam/ft_parser.c @@ -0,0 +1,152 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +#ifdef EVAL_RUN +#ifdef PIVOT_STAT +ulong collstat=0; +#endif +#endif /* EVAL_RUN */ + +typedef struct st_ft_docstat { + FT_WORD *list; + uint uniq; + double sum; +#ifdef EVAL_RUN + uint words, totlen; + double max, nsum, nsum2; +#endif /* EVAL_RUN */ + + MI_INFO *info; + uint keynr; + byte *keybuf; +} FT_DOCSTAT; + +static int FT_WORD_cmp(FT_WORD *w1, FT_WORD *w2) +{ + return _mi_compare_text(default_charset_info, + (uchar*) w1->pos,w1->len, + (uchar*) w2->pos, w2->len,0); +} + +static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) +{ + if(is_stopword(word->pos, word->len)) + return 0; + + word->weight=LWS_IN_USE; + +#ifdef EVAL_RUN + word->cnt= (uchar) count; + if(docstat->max < word->weight) docstat->max=word->weight; + docstat->words+=count; + docstat->totlen+=word->len; +#endif /* EVAL_RUN */ + docstat->sum+=word->weight; + + memcpy_fixed((docstat->list)++,word,sizeof(FT_WORD)); + return 0; +} + +/* transforms tree of words into the array, applying normalization */ + +FT_WORD * ft_linearize(MI_INFO *info, uint keynr, byte *keybuf, TREE *wtree) +{ + FT_WORD *wlist,*p; + FT_DOCSTAT docstat; + + if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)* + (1+wtree->elements_in_tree),MYF(0)))) + { + docstat.info=info; + docstat.keynr=keynr; + docstat.keybuf=keybuf; + docstat.list=wlist; + docstat.uniq=wtree->elements_in_tree; +#ifdef EVAL_RUN + docstat.nsum=docstat.nsum2=docstat.max=docstat.words=docstat.totlen= +#endif /* EVAL_RUN */ + docstat.sum=0; + tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right); + } + delete_tree(wtree); + free(wtree); + if(wlist==NULL) + return NULL; + + docstat.list->pos=NULL; + + for(p=wlist;p->pos;p++) + { + p->weight=PRENORM_IN_USE; +#ifdef EVAL_RUN + docstat.nsum+=p->weight; + docstat.nsum2+=p->weight*p->weight; +#endif /* EVAL_RUN */ + } + +#ifdef EVAL_RUN +#ifdef PIVOT_STAT + collstat+=PIVOT_STAT; +#endif +#endif /* EVAL_RUN */ + + for(p=wlist;p->pos;p++) + { + p->weight/=NORM_IN_USE; + } + + return wlist; +} + +#ifdef HYPHEN_IS_DELIM +#define word_char(X) (isalnum(X) || (X)=='_' || (X)=='\'') +#else +#define word_char(X) (isalnum(X) || (X)=='_' || (X)=='\'' || (X)=='-') +#endif + +/* this is rather dumb first version of the parser */ +TREE * ft_parse(TREE *wtree, byte *doc, int doclen) +{ + byte *end=doc+doclen; + FT_WORD w; + + if(!wtree) + { + if(!(wtree=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) return NULL; + init_tree(wtree,0,sizeof(FT_WORD),(qsort_cmp)&FT_WORD_cmp,0,NULL); + } + + w.weight=0; + while(doc<end) + { + for(;doc<end;doc++) + if(word_char(*doc)) break; + for(w.pos=doc; doc<end; doc++) + if(!word_char(*doc)) break; + if((w.len=doc-w.pos) < MIN_WORD_LEN) continue; + if(!tree_insert(wtree, &w, 0)) + { + delete_tree(wtree); + free(wtree); + return NULL; + } + } + return wtree; +} diff --git a/myisam/ft_search.c b/myisam/ft_search.c new file mode 100644 index 00000000000..db3afd3057b --- /dev/null +++ b/myisam/ft_search.c @@ -0,0 +1,223 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +/* queries isam and returns list of documents matched */ + +typedef struct st_all_in_one { + MI_INFO *info; + uint keynr; + uchar *keybuff; + MI_KEYDEF *keyinfo; + my_off_t key_root; + TREE dtree; +} ALL_IN_ONE; + +typedef struct st_ft_superdoc { + FT_DOC doc; + FT_WORD *word_ptr; + double tmp_weight; +} FT_SUPERDOC; + +static int FT_SUPERDOC_cmp(FT_SUPERDOC *p1, FT_SUPERDOC *p2) +{ + if (p1->doc.dpos < p2->doc.dpos) + return -1; + if (p1->doc.dpos == p2->doc.dpos) + return 0; + return 1; +} + +static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) +{ + uint keylen, r, doc_cnt; +#ifdef EVAL_RUN + uint cnt; + double sum, sum2, suml; +#endif /* EVAL_RUN */ + FT_SUPERDOC sdoc, *sptr; + TREE_ELEMENT *selem; +#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT + float tmp_weight; +#else +#error +#endif + + word->weight=LWS_FOR_QUERY; + + keylen=_ft_make_key(aio->info,aio->keynr,(char*) aio->keybuff,word,0); +#ifdef EVAL_RUN + keylen-=1+HA_FT_WLEN; +#else /* EVAL_RUN */ + keylen-=HA_FT_WLEN; +#endif /* EVAL_RUN */ + +#ifdef EVAL_RUN + sum=sum2=suml= +#endif /* EVAL_RUN */ + doc_cnt=0; + + r=_mi_search(aio->info, aio->keyinfo, aio->keybuff, keylen, + SEARCH_FIND | SEARCH_PREFIX, aio->key_root); + + while(!r) + { + if (_mi_compare_text(default_charset_info, + aio->info->lastkey,keylen, + aio->keybuff,keylen,0)) break; + +#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT +#ifdef EVAL_RUN + mi_float4get(tmp_weight,aio->info->lastkey+keylen+1); +#else /* EVAL_RUN */ + mi_float4get(tmp_weight,aio->info->lastkey+keylen); +#endif /* EVAL_RUN */ +#else +#error +#endif + if(tmp_weight==0) return doc_cnt; /* stopword, doc_cnt should be 0 */ + +#ifdef EVAL_RUN + cnt=*(byte *)(aio->info->lastkey+keylen); +#endif /* EVAL_RUN */ + + sdoc.doc.dpos=aio->info->lastpos; + + /* saving document matched into dtree */ + if(!(selem=tree_insert(&aio->dtree, &sdoc, 0))) return 1; + + sptr=(FT_SUPERDOC *)ELEMENT_KEY((&aio->dtree), selem); + + if(selem->count==1) /* document's first match */ + sptr->doc.weight=0; + else + sptr->doc.weight+=sptr->tmp_weight*sptr->word_ptr->weight; + + sptr->word_ptr=word; + sptr->tmp_weight=tmp_weight; + + doc_cnt++; +#ifdef EVAL_RUN + sum +=cnt; + sum2+=cnt*cnt; + suml+=cnt*log(cnt); +#endif /* EVAL_RUN */ + + if (_mi_test_if_changed(aio->info) == 0) + r=_mi_search_next(aio->info, aio->keyinfo, aio->info->lastkey, + aio->info->lastkey_length, SEARCH_BIGGER, + aio->key_root); + else + r=_mi_search(aio->info, aio->keyinfo, aio->info->lastkey, + aio->info->lastkey_length, SEARCH_BIGGER, + aio->key_root); + } + if(doc_cnt) { + word->weight*=GWS_IN_USE; + if(word->weight < 0) word->weight=0; + } + + return 0; +} + +static int walk_and_copy(FT_SUPERDOC *from, + uint32 count __attribute__((unused)), FT_DOC **to) +{ + from->doc.weight+=from->tmp_weight*from->word_ptr->weight; + (*to)->dpos=from->doc.dpos; + (*to)->weight=from->doc.weight; + (*to)++; + return 0; +} + +static int FT_DOC_cmp(FT_DOC *a, FT_DOC *b) +{ + return sgn(b->weight - a->weight); +} + +FT_DOCLIST * ft_init_search(void *info, uint keynr, byte *key, + uint key_len, my_bool presort) +{ + TREE *wtree; + ALL_IN_ONE aio; + FT_DOCLIST *dlist; + FT_DOC *dptr; + +/* black magic ON */ + if ((int) (keynr = _mi_check_index((MI_INFO *)info,keynr)) < 0) + return NULL; + if (_mi_readinfo((MI_INFO *)info,F_RDLCK,1)) + return NULL; +/* black magic OFF */ + + dlist=NULL; + aio.info=(MI_INFO *)info; + aio.keynr=keynr; + aio.keybuff=aio.info->lastkey+aio.info->s->base.max_key_length; + aio.keyinfo=aio.info->s->keyinfo+keynr; + aio.key_root=aio.info->s->state.key_root[keynr]; + + if(!(wtree=ft_parse(NULL,key,key_len))) return NULL; + + init_tree(&aio.dtree,0,sizeof(FT_SUPERDOC),(qsort_cmp)&FT_SUPERDOC_cmp,0, + NULL); + + if(tree_walk(wtree, (tree_walk_action)&walk_and_match, &aio, + left_root_right)) + goto err; + + dlist=(FT_DOCLIST *)my_malloc(sizeof(FT_DOCLIST)+sizeof(FT_DOC)*(aio.dtree.elements_in_tree-1),MYF(0)); + if(!dlist) + goto err; + + dlist->ndocs=aio.dtree.elements_in_tree; + dlist->curdoc=0; + dlist->info=aio.info; + dptr=dlist->doc; + + tree_walk(&aio.dtree, (tree_walk_action)&walk_and_copy, &dptr, left_root_right); + + if(presort) + { + qsort(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort_cmp)&FT_DOC_cmp); + } + +err: + delete_tree(&aio.dtree); + delete_tree(wtree); + free(wtree); + return dlist; +} + +double ft_read_next(FT_DOCLIST *handler, char *record) +{ + MI_INFO *info=handler->info; + + if (handler->curdoc >= handler->ndocs) + return 0; + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (!(*info->read_record)(info,handler->doc[handler->curdoc].dpos,record)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + return handler->doc[handler->curdoc++].weight; + } + return -my_errno; +} diff --git a/myisam/ft_static.c b/myisam/ft_static.c new file mode 100644 index 00000000000..5cbcff85b54 --- /dev/null +++ b/myisam/ft_static.c @@ -0,0 +1,627 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +const MI_KEYSEG ft_keysegs[FT_SEGS]={ + { + HA_KEYTYPE_VARTEXT, // type + 7, // language + 0, 0, 0, // null_bit, bit_start, bit_end + HA_VAR_LENGTH | HA_PACK_KEY, // flag + HA_FT_MAXLEN, // length +#ifdef EVAL_RUN + HA_FT_WLEN+1, // start +#else /* EVAL_RUN */ + HA_FT_WLEN, // start +#endif /* EVAL_RUN */ + 0, // null_pos + NULL // sort_order + }, +#ifdef EVAL_RUN + { + HA_KEYTYPE_INT8, 7, 0, 0, 0, 0, 1, HA_FT_WLEN, 0, NULL + }, +#endif /* EVAL_RUN */ + { + HA_FT_WTYPE, 7, 0, 0, 0, 0, HA_FT_WLEN, 0, 0, NULL + } +}; + +const char *ft_precompiled_stopwords[] = { + +#ifdef COMPILE_STOPWORDS_IN + +/* This particular stopword list was taken from SMART distribution + ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z + it was slightly modified to my taste, though + */ + + "a", + "a's", + "able", + "about", + "above", + "according", + "accordingly", + "across", + "actually", + "after", + "afterwards", + "again", + "against", + "ain't", + "all", + "allow", + "allows", + "almost", + "alone", + "along", + "already", + "also", + "although", + "always", + "am", + "among", + "amongst", + "an", + "and", + "another", + "any", + "anybody", + "anyhow", + "anyone", + "anything", + "anyway", + "anyways", + "anywhere", + "apart", + "appear", + "appreciate", + "appropriate", + "are", + "aren't", + "around", + "as", + "aside", + "ask", + "asking", + "associated", + "at", + "available", + "away", + "awfully", + "b", + "be", + "became", + "because", + "become", + "becomes", + "becoming", + "been", + "before", + "beforehand", + "behind", + "being", + "believe", + "below", + "beside", + "besides", + "best", + "better", + "between", + "beyond", + "both", + "brief", + "but", + "by", + "c", + "c'mon", + "c's", + "came", + "can", + "can't", + "cannot", + "cant", + "cause", + "causes", + "certain", + "certainly", + "changes", + "clearly", + "co", + "com", + "come", + "comes", + "concerning", + "consequently", + "consider", + "considering", + "contain", + "containing", + "contains", + "corresponding", + "could", + "couldn't", + "course", + "currently", + "d", + "definitely", + "described", + "despite", + "did", + "didn't", + "different", + "do", + "does", + "doesn't", + "doing", + "don't", + "done", + "down", + "downwards", + "during", + "e", + "each", + "edu", + "eg", + "eight", + "either", + "else", + "elsewhere", + "enough", + "entirely", + "especially", + "et", + "etc", + "even", + "ever", + "every", + "everybody", + "everyone", + "everything", + "everywhere", + "ex", + "exactly", + "example", + "except", + "f", + "far", + "few", + "fifth", + "first", + "five", + "followed", + "following", + "follows", + "for", + "former", + "formerly", + "forth", + "four", + "from", + "further", + "furthermore", + "g", + "get", + "gets", + "getting", + "given", + "gives", + "go", + "goes", + "going", + "gone", + "got", + "gotten", + "greetings", + "h", + "had", + "hadn't", + "happens", + "hardly", + "has", + "hasn't", + "have", + "haven't", + "having", + "he", + "he's", + "hello", + "help", + "hence", + "her", + "here", + "here's", + "hereafter", + "hereby", + "herein", + "hereupon", + "hers", + "herself", + "hi", + "him", + "himself", + "his", + "hither", + "hopefully", + "how", + "howbeit", + "however", + "i", + "i'd", + "i'll", + "i'm", + "i've", + "ie", + "if", + "ignored", + "immediate", + "in", + "inasmuch", + "inc", + "indeed", + "indicate", + "indicated", + "indicates", + "inner", + "insofar", + "instead", + "into", + "inward", + "is", + "isn't", + "it", + "it'd", + "it'll", + "it's", + "its", + "itself", + "j", + "just", + "k", + "keep", + "keeps", + "kept", + "know", + "knows", + "known", + "l", + "last", + "lately", + "later", + "latter", + "latterly", + "least", + "less", + "lest", + "let", + "let's", + "like", + "liked", + "likely", + "little", + "look", + "looking", + "looks", + "ltd", + "m", + "mainly", + "many", + "may", + "maybe", + "me", + "mean", + "meanwhile", + "merely", + "might", + "more", + "moreover", + "most", + "mostly", + "much", + "must", + "my", + "myself", + "n", + "name", + "namely", + "nd", + "near", + "nearly", + "necessary", + "need", + "needs", + "neither", + "never", + "nevertheless", + "new", + "next", + "nine", + "no", + "nobody", + "non", + "none", + "noone", + "nor", + "normally", + "not", + "nothing", + "novel", + "now", + "nowhere", + "o", + "obviously", + "of", + "off", + "often", + "oh", + "ok", + "okay", + "old", + "on", + "once", + "one", + "ones", + "only", + "onto", + "or", + "other", + "others", + "otherwise", + "ought", + "our", + "ours", + "ourselves", + "out", + "outside", + "over", + "overall", + "own", + "p", + "particular", + "particularly", + "per", + "perhaps", + "placed", + "please", + "plus", + "possible", + "presumably", + "probably", + "provides", + "q", + "que", + "quite", + "qv", + "r", + "rather", + "rd", + "re", + "really", + "reasonably", + "regarding", + "regardless", + "regards", + "relatively", + "respectively", + "right", + "s", + "said", + "same", + "saw", + "say", + "saying", + "says", + "second", + "secondly", + "see", + "seeing", + "seem", + "seemed", + "seeming", + "seems", + "seen", + "self", + "selves", + "sensible", + "sent", + "serious", + "seriously", + "seven", + "several", + "shall", + "she", + "should", + "shouldn't", + "since", + "six", + "so", + "some", + "somebody", + "somehow", + "someone", + "something", + "sometime", + "sometimes", + "somewhat", + "somewhere", + "soon", + "sorry", + "specified", + "specify", + "specifying", + "still", + "sub", + "such", + "sup", + "sure", + "t", + "t's", + "take", + "taken", + "tell", + "tends", + "th", + "than", + "thank", + "thanks", + "thanx", + "that", + "that's", + "thats", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "thence", + "there", + "there's", + "thereafter", + "thereby", + "therefore", + "therein", + "theres", + "thereupon", + "these", + "they", + "they'd", + "they'll", + "they're", + "they've", + "think", + "third", + "this", + "thorough", + "thoroughly", + "those", + "though", + "three", + "through", + "throughout", + "thru", + "thus", + "to", + "together", + "too", + "took", + "toward", + "towards", + "tried", + "tries", + "truly", + "try", + "trying", + "twice", + "two", + "u", + "un", + "under", + "unfortunately", + "unless", + "unlikely", + "until", + "unto", + "up", + "upon", + "us", + "use", + "used", + "useful", + "uses", + "using", + "usually", + "v", + "value", + "various", + "very", + "via", + "viz", + "vs", + "w", + "want", + "wants", + "was", + "wasn't", + "way", + "we", + "we'd", + "we'll", + "we're", + "we've", + "welcome", + "well", + "went", + "were", + "weren't", + "what", + "what's", + "whatever", + "when", + "whence", + "whenever", + "where", + "where's", + "whereafter", + "whereas", + "whereby", + "wherein", + "whereupon", + "wherever", + "whether", + "which", + "while", + "whither", + "who", + "who's", + "whoever", + "whole", + "whom", + "whose", + "why", + "will", + "willing", + "wish", + "with", + "within", + "without", + "won't", + "wonder", + "would", + "would", + "wouldn't", + "x", + "y", + "yes", + "yet", + "you", + "you'd", + "you'll", + "you're", + "you've", + "your", + "yours", + "yourself", + "yourselves", + "z", + "zero", +#endif + + NULL }; diff --git a/myisam/ft_stem.c b/myisam/ft_stem.c new file mode 100644 index 00000000000..bdfc73b774b --- /dev/null +++ b/myisam/ft_stem.c @@ -0,0 +1,20 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* mulitingual stem */ + diff --git a/myisam/ft_stopwords.c b/myisam/ft_stopwords.c new file mode 100644 index 00000000000..f8fddc9d4bb --- /dev/null +++ b/myisam/ft_stopwords.c @@ -0,0 +1,68 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +typedef struct st_ft_stopwords { + const char * pos; + uint len; +} FT_STOPWORD; + +static TREE *stopwords3=NULL; + +static int FT_STOPWORD_cmp(FT_STOPWORD *w1, FT_STOPWORD *w2) +{ + return _mi_compare_text(default_charset_info, + (uchar *)w1->pos,w1->len, + (uchar *)w2->pos,w2->len,0); +} + +int ft_init_stopwords(const char **sws) +{ + FT_STOPWORD sw; + + + if(!stopwords3) + { + if(!(stopwords3=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) return -1; + init_tree(stopwords3,0,sizeof(FT_STOPWORD),(qsort_cmp)&FT_STOPWORD_cmp,0, + NULL); + } + + if(!sws) return 0; + + for(;*sws;sws++) + { + if( (sw.len=strlen(sw.pos=*sws)) < MIN_WORD_LEN) continue; + if(!tree_insert(stopwords3, &sw, 0)) + { + delete_tree(stopwords3); + return -1; + } + } + return 0; +} + +int is_stopword(char *word, uint len) +{ + FT_STOPWORD sw; + sw.pos=word; + sw.len=len; + return tree_search(stopwords3,&sw) != NULL; +} + diff --git a/myisam/ft_test1.c b/myisam/ft_test1.c new file mode 100644 index 00000000000..377fa808d52 --- /dev/null +++ b/myisam/ft_test1.c @@ -0,0 +1,265 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" +#include "ft_test1.h" +#include <getopt.h> + +static int key_field=FIELD_VARCHAR,extra_field=FIELD_SKIPP_ENDSPACE; +static uint key_length=200,extra_length=50; +static int key_type=HA_KEYTYPE_TEXT; +static int verbose=0,silent=0,skip_update=0, + no_keys=0,no_stopwords=0,no_search=0,no_fulltext=0; +static int create_flag=0,error=0; + +#define MAX_REC_LENGTH 300 +static char record[MAX_REC_LENGTH],read_record[MAX_REC_LENGTH]; + +void get_options(int argc,char *argv[]); +static int run_test(const char *filename); + +int main(int argc,char *argv[]) +{ + MY_INIT(argv[0]); + + get_options(argc,argv); + + exit(run_test("FT1")); +} + +static MI_COLUMNDEF recinfo[3]; +static MI_KEYDEF keyinfo[2]; +static MI_KEYSEG keyseg[10]; + +void create_record(char *, int); + +static int run_test(const char *filename) +{ + MI_INFO *file; + int i,j; + my_off_t pos; + + bzero((char*) recinfo,sizeof(recinfo)); + + /* First define 2 columns */ + recinfo[0].type=extra_field; + recinfo[0].length= (extra_field == FIELD_BLOB ? 4 + mi_portable_sizeof_char_ptr : + extra_length); + if (extra_field == FIELD_VARCHAR) + recinfo[0].length+=2; + recinfo[1].type=key_field; + recinfo[1].length= (key_field == FIELD_BLOB ? 4+mi_portable_sizeof_char_ptr : + key_length); + if (key_field == FIELD_VARCHAR) + recinfo[1].length+=2; + + /* Define a key over the first column */ + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=1; + keyinfo[0].seg[0].type= key_type; + keyinfo[0].seg[0].flag= (key_field == FIELD_BLOB)?HA_BLOB_PART: + (key_field == FIELD_VARCHAR)?HA_VAR_LENGTH:0; + keyinfo[0].seg[0].start=recinfo[0].length; + keyinfo[0].seg[0].length=key_length; + keyinfo[0].seg[0].null_bit= 0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[0].flag = (no_fulltext?HA_PACK_KEY:HA_FULLTEXT); + + if (!silent) + printf("- Creating isam-file\n"); + if (mi_create(filename,(no_keys?0:1),keyinfo,2,recinfo,0,NULL, + (MI_CREATE_INFO*) 0, create_flag)) + goto err; + if (!(file=mi_open(filename,2,0))) + goto err; + + if (!silent) + printf("- %s stopwords\n",no_stopwords?"Skipping":"Initializing"); + ft_init_stopwords(no_stopwords?NULL:ft_precompiled_stopwords); + + if (!silent) + printf("- Writing key:s\n"); + + my_errno=0; + for (i=NUPD ; i<NDATAS; i++ ) + { + create_record(record,i); + error=mi_write(file,record); + if (verbose || error) + printf("I= %2d mi_write: %d errno: %d, record: %s\n", + i,error,my_errno,data[i].f0); + } + + if (!skip_update) + { + if (!silent) + printf("- Updating rows\n"); + + /* Read through all rows and update them */ + pos=(ha_rows) 0; + i=0; + while ((error=mi_rrnd(file,read_record,pos)) == 0) + { + create_record(record,NUPD-i-1); + if (mi_update(file,read_record,record)) + { + printf("Can't update row: %.*s, error: %d\n", + keyinfo[0].seg[0].length,record,my_errno); + } + if(++i == NUPD) break; + pos=HA_OFFSET_ERROR; + } + if (i != NUPD) + printf("Found %d of %d rows\n", i,NUPD); + } + + if (mi_close(file)) goto err; + if(no_search) return 0; + if (!silent) + printf("- Reopening file\n"); + if (!(file=mi_open(filename,2,0))) goto err; + if (!silent) + printf("- Reading rows with key\n"); + for (i=0 ; i < NQUERIES ; i++) + { FT_DOCLIST *result; + result=ft_init_search(file,0,(char*) query[i],strlen(query[i]),1); + if(!result) { + printf("Query %d: `%s' failed with errno %3d\n",i,query[i],my_errno); + continue; + } + printf("Query %d: `%s'. Found: %d. Top five documents:\n", + i,query[i],result->ndocs); + for(j=0;j<5;j++) { double w; + w=ft_read_next(result, read_record); + if(w<0) { + printf("ft_read_next %d failed with errno %3d\n",j,my_errno); + break; + } else if (w==0) { + printf("No more matches!\n"); + break; + } + if(key_field == FIELD_VARCHAR) { + uint l; + char *p; + p=recinfo[0].length+read_record; + l=uint2korr(p); + printf("%10.7f: %.*s\n",w,(int) l,p+2); + } else + printf("%10.7f: %.*s\n",w,recinfo[1].length, + recinfo[0].length+read_record); + } + ft_close_search(result); + } + + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + + return (0); +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skipp warning */ +} + +static char blob_key[MAX_REC_LENGTH]; +/* static char blob_record[MAX_REC_LENGTH+20*20]; */ + +void create_record(char *pos, int n) +{ + bzero((char*) pos,MAX_REC_LENGTH); + if (recinfo[0].type == FIELD_BLOB) + { + uint tmp; + char *ptr; + strncpy(blob_key,data[n].f0,keyinfo[0].seg[0].length); + tmp=strlen(blob_key); + int4store(pos,tmp); + ptr=blob_key; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); + pos+=recinfo[0].length; + } + else if (recinfo[0].type == FIELD_VARCHAR) + { + uint tmp; + strncpy(pos+2,data[n].f0,keyinfo[0].seg[0].length); + tmp=strlen(pos+2); + int2store(pos,tmp); + pos+=recinfo[0].length; + } + else + { + strncpy(pos,data[n].f0,keyinfo[0].seg[0].length); + pos+=recinfo[0].length; + } + if (recinfo[1].type == FIELD_BLOB) + { + uint tmp; + char *ptr; + strncpy(blob_key,data[n].f2,keyinfo[0].seg[0].length); + tmp=strlen(blob_key); + int4store(pos,tmp); + ptr=blob_key; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); + pos+=recinfo[1].length; + } + else if (recinfo[1].type == FIELD_VARCHAR) + { + uint tmp; + strncpy(pos+2,data[n].f2,keyinfo[0].seg[0].length); + tmp=strlen(pos+2); + int2store(pos,tmp); + pos+=recinfo[1].length; + } + else + { + strncpy(pos,data[n].f2,keyinfo[0].seg[0].length); + pos+=recinfo[1].length; + } +} + +/* Read options */ + +void get_options(int argc,char *argv[]) +{ + int c; + const char *options="hVvsNSKFU#:"; + + while ((c=getopt(argc,argv,options)) != -1) + { + switch(c) { + case 'v': verbose=1; break; + case 's': silent=1; break; + case 'F': no_fulltext=1; no_search=1; + case 'U': skip_update=1; break; + case 'K': no_keys=no_search=1; break; + case 'N': no_search=1; break; + case 'S': no_stopwords=1; break; + case '#': + DEBUGGER_ON; + DBUG_PUSH (optarg); + break; + case 'V': + case '?': + case 'h': + default: + printf("%s -[%s]\n", argv[0], options); + exit(0); + } + } + return; +} /* get options */ diff --git a/myisam/ft_test1.h b/myisam/ft_test1.h new file mode 100644 index 00000000000..17b0cae66b7 --- /dev/null +++ b/myisam/ft_test1.h @@ -0,0 +1,422 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#define NUPD 20 +#define NDATAS 389 +struct { const char *f0, *f2; } data[NDATAS] = { + {"1", "General Information about MySQL"}, + {"1.1", "What is MySQL?"}, + {"1.2", "About this manual"}, + {"1.3", "History of MySQL"}, + {"1.4", "The main features of MySQL"}, + {"1.5", "General SQL information and tutorials"}, + {"1.6", "Useful MySQL-related links"}, + {"1.7", "What are stored procedures and triggers and so on?"}, + {"2", "MySQL mailing lists and how to ask questions/give error (bug) reports"}, + {"2.1", "Subscribing to/un-subscribing from the MySQL mailing list"}, + {"2.2", "Asking questions or reporting bugs"}, + {"2.3", "I think I have found a bug. What information do you need to help me?"}, + {"2.3.1", "MySQL keeps crashing"}, + {"2.4", "Guidelines for answering questions on the mailing list"}, + {"3", "Licensing or When do I have/want to pay for MySQL?"}, + {"3.1", "How much does MySQL cost?"}, + {"3.2", "How do I get commercial support?"}, + {"3.2.1", "Types of commercial support"}, + {"3.2.1.1", "Basic email support"}, + {"3.2.1.2", "Extended email support"}, +/*------------------------------- NUPD=20 -------------------------------*/ + {"3.2.1.3", "Asking: Login support"}, + {"3.2.1.4", "Extended login support"}, + {"3.3", "How do I pay for licenses/support?"}, + {"3.4", "Who do I contact when I want more information about licensing/support?"}, + {"3.5", "What Copyright does MySQL use?"}, + {"3.6", "When may I distribute MySQL commercially without a fee?"}, + {"3.7", "I want to sell a product that can be configured to use MySQL"}, + {"3.8", "I am running a commercial web server using MySQL"}, + {"3.9", "Do I need a license to sell commercial Perl/tcl/PHP/Web+ etc applications?"}, + {"3.10", "Possible future changes in the licensing"}, + {"4", "Compiling and installing MySQL"}, + {"4.1", "How do I get MySQL?"}, + {"4.2", "Which MySQL version should I use?"}, + {"4.3", "How/when will you release updates?"}, + {"4.4", "What operating systems does MySQL support?"}, + {"4.5", "Compiling MySQL from source code"}, + {"4.5.1", "Quick installation overview"}, + {"4.5.2", "Usual configure switches"}, + {"4.5.3", "Applying a patch"}, + {"4.6", "Problems compiling?"}, + {"4.7", "General compilation notes"}, + {"4.8", "MIT-pthreads notes (FreeBSD)"}, + {"4.9", "Perl installation comments"}, + {"4.10", "Special things to consider for some machine/OS combinations"}, + {"4.10.1", "Solaris notes"}, + {"4.10.2", "SunOS 4 notes"}, + {"4.10.3", "Linux notes for all versions"}, + {"4.10.3.1", "Linux-x86 notes"}, + {"4.10.3.2", "RedHat 5.0"}, + {"4.10.3.3", "RedHat 5.1"}, + {"4.10.3.4", "Linux-Sparc notes"}, + {"4.10.3.5", "Linux-Alpha notes"}, + {"4.10.3.6", "MkLinux notes"}, + {"4.10.4", "Alpha-DEC-Unix notes"}, + {"4.10.5", "Alpha-DEC-OSF1 notes"}, + {"4.10.6", "SGI-IRIX notes"}, + {"4.10.7", "FreeBSD notes"}, + {"4.10.7.1", "FreeBSD-3.0 notes"}, + {"4.10.8", "BSD/OS 2.# notes"}, + {"4.10.8.1", "BSD/OS 3.# notes"}, + {"4.10.9", "SCO notes"}, + {"4.10.10", "SCO Unixware 7.0 notes"}, + {"4.10.11", "IBM-AIX notes"}, + {"4.10.12", "HP-UX notes"}, + {"4.11", "TcX binaries"}, + {"4.12", "Win32 notes"}, + {"4.13", "Installation instructions for MySQL binary releases"}, + {"4.13.1", "How to get MySQL Perl support working"}, + {"4.13.2", "Linux notes"}, + {"4.13.3", "HP-UX notes"}, + {"4.13.4", "Linking client libraries"}, + {"4.14", "Problems running mysql_install_db"}, + {"4.15", "Problems starting MySQL"}, + {"4.16", "Automatic start/stop of MySQL"}, + {"4.17", "Option files"}, + {"5", "How standards-compatible is MySQL?"}, + {"5.1", "What extensions has MySQL to ANSI SQL92?"}, + {"5.2", "What functionality is missing in MySQL?"}, + {"5.2.1", "Sub-selects"}, + {"5.2.2", "SELECT INTO TABLE"}, + {"5.2.3", "Transactions"}, + {"5.2.4", "Triggers"}, + {"5.2.5", "Foreign Keys"}, + {"5.2.5.1", "Some reasons NOT to use FOREIGN KEYS"}, + {"5.2.6", "Views"}, + {"5.2.7", "-- as start of a comment"}, + {"5.3", "What standards does MySQL follow?"}, + {"5.4", "What functions exist only for compatibility?"}, + {"5.5", "Limitations of BLOB and TEXT types"}, + {"5.6", "How to cope without COMMIT-ROLLBACK"}, + {"6", "The MySQL access privilege system"}, + {"6.1", "What the privilege system does"}, + {"6.2", "Connecting to the MySQL server"}, + {"6.2.1", "Keeping your password secure"}, + {"6.3", "Privileges provided by MySQL"}, + {"6.4", "How the privilege system works"}, + {"6.5", "The privilege tables"}, + {"6.6", "Setting up the initial MySQL privileges"}, + {"6.7", "Adding new user privileges to MySQL"}, + {"6.8", "An example permission setup"}, + {"6.9", "Causes of Access denied errors"}, + {"6.10", "How to make MySQL secure against crackers"}, + {"7", "MySQL language reference"}, + {"7.1", "Literals: how to write strings and numbers"}, + {"7.1.1", "Strings"}, + {"7.1.2", "Numbers"}, + {"7.1.3", "NULL values"}, + {"7.1.4", "Database, table, index, column and alias names"}, + {"7.1.4.1", "Case sensitivity in names"}, + {"7.2", "Column types"}, + {"7.2.1", "Column type storage requirements"}, + {"7.2.5", "Numeric types"}, + {"7.2.6", "Date and time types"}, + {"7.2.6.1", "The DATE type"}, + {"7.2.6.2", "The TIME type"}, + {"7.2.6.3", "The DATETIME type"}, + {"7.2.6.4", "The TIMESTAMP type"}, + {"7.2.6.5", "The YEAR type"}, + {"7.2.6.6", "Miscellaneous date and time properties"}, + {"7.2.7", "String types"}, + {"7.2.7.1", "The CHAR and VARCHAR types"}, + {"7.2.7.2", "The BLOB and TEXT types"}, + {"7.2.7.3", "The ENUM type"}, + {"7.2.7.4", "The SET type"}, + {"7.2.8", "Choosing the right type for a column"}, + {"7.2.9", "Column indexes"}, + {"7.2.10", "Multiple-column indexes"}, + {"7.2.11", "Using column types from other database engines"}, + {"7.3", "Functions for use in SELECT and WHERE clauses"}, + {"7.3.1", "Grouping functions"}, + {"7.3.2", "Normal arithmetic operations"}, + {"7.3.3", "Bit functions"}, + {"7.3.4", "Logical operations"}, + {"7.3.5", "Comparison operators"}, + {"7.3.6", "String comparison functions"}, + {"7.3.7", "Control flow functions"}, + {"7.3.8", "Mathematical functions"}, + {"7.3.9", "String functions"}, + {"7.3.10", "Date and time functions"}, + {"7.3.11", "Miscellaneous functions"}, + {"7.3.12", "Functions for use with GROUP BY clauses"}, + {"7.4", "CREATE DATABASE syntax"}, + {"7.5", "DROP DATABASE syntax"}, + {"7.6", "CREATE TABLE syntax"}, + {"7.7", "ALTER TABLE syntax"}, + {"7.8", "OPTIMIZE TABLE syntax"}, + {"7.9", "DROP TABLE syntax"}, + {"7.10", "DELETE syntax"}, + {"7.11", "SELECT syntax"}, + {"7.12", "JOIN syntax"}, + {"7.13", "INSERT syntax"}, + {"7.14", "REPLACE syntax"}, + {"7.15", "LOAD DATA INFILE syntax"}, + {"7.16", "UPDATE syntax"}, + {"7.17", "USE syntax"}, + {"7.18", "SHOW syntax (Get information about tables, columns...)"}, + {"7.19", "EXPLAIN syntax (Get information about a SELECT)"}, + {"7.20", "DESCRIBE syntax (Get information about columns)"}, + {"7.21", "LOCK TABLES/UNLOCK TABLES syntax"}, + {"7.22", "SET OPTION syntax"}, + {"7.23", "GRANT syntax (Compatibility function)"}, + {"7.24", "CREATE INDEX syntax (Compatibility function)"}, + {"7.25", "DROP INDEX syntax (Compatibility function)"}, + {"7.26", "Comment syntax"}, + {"7.27", "CREATE FUNCTION/DROP FUNCTION syntax"}, + {"7.28", "Is MySQL picky about reserved words?"}, + {"8", "Example SQL queries"}, + {"8.1", "Queries from twin project"}, + {"8.1.1", "Find all non-distributed twins"}, + {"8.1.2", "Show a table on twin pair status"}, + {"9", "How safe/stable is MySQL?"}, + {"9.1", "How stable is MySQL?"}, + {"9.2", "Why are there is so many releases of MySQL?"}, + {"9.3", "Checking a table for errors"}, + {"9.4", "How to repair tables"}, + {"9.5", "Is there anything special to do when upgrading/downgrading MySQL?"}, + {"9.5.1", "Upgrading from a 3.21 version to 3.22"}, + {"9.5.2", "Upgrading from a 3.20 version to 3.21"}, + {"9.5.3", "Upgrading to another architecture"}, + {"9.6", "Year 2000 compliance"}, + {"10", "MySQL Server functions"}, + {"10.1", "What languages are supported by MySQL?"}, + {"10.1.1", "Character set used for data & sorting"}, + {"10.2", "The update log"}, + {"10.3", "How big can MySQL tables be?"}, + {"11", "Getting maximum performance from MySQL"}, + {"11.1", "How does one change the size of MySQL buffers?"}, + {"11.2", "How compiling and linking affects the speed of MySQL"}, + {"11.3", "How does MySQL use memory?"}, + {"11.4", "How does MySQL use indexes?"}, + {"11.5", "What optimizations are done on WHERE clauses?"}, + {"11.6", "How does MySQL open & close tables?"}, + {"11.6.0.1", "What are the drawbacks of creating possibly thousands of tables in a database?"}, + {"11.7", "How does MySQL lock tables?"}, + {"11.8", "How should I arrange my table to be as fast/small as possible?"}, + {"11.9", "What affects the speed of INSERT statements?"}, + {"11.10", "What affects the speed DELETE statements?"}, + {"11.11", "How do I get MySQL to run at full speed?"}, + {"11.12", "What are the different row formats? Or, when should VARCHAR/CHAR be used?"}, + {"11.13", "Why so many open tables?"}, + {"12", "MySQL benchmark suite"}, + {"13", "MySQL Utilites"}, + {"13.1", "Overview of the different MySQL programs"}, + {"13.2", "The MySQL table check, optimize and repair program"}, + {"13.2.1", "isamchk memory use"}, + {"13.2.2", "Getting low-level table information"}, + {"13.3", "The MySQL compressed read-only table generator"}, + {"14", "Adding new functions to MySQL"}, + {"15", "MySQL ODBC Support"}, + {"15.1", "Operating systems supported by MyODBC"}, + {"15.2", "How to report problems with MyODBC"}, + {"15.3", "Programs known to work with MyODBC"}, + {"15.4", "How to fill in the various fields in the ODBC administrator program"}, + {"15.5", "How to get the value of an AUTO_INCREMENT column in ODBC"}, + {"16", "Problems and common errors"}, + {"16.1", "Some common errors when using MySQL"}, + {"16.1.1", "MySQL server has gone away error"}, + {"16.1.2", "Can't connect to local MySQL server error"}, + {"16.1.3", "Out of memory error"}, + {"16.1.4", "Packet too large error"}, + {"16.1.5", "The table is full error"}, + {"16.1.6", "Commands out of sync error in client"}, + {"16.1.7", "Removing user error"}, + {"16.2", "How MySQL handles a full disk"}, + {"16.3", "How to run SQL commands from a text file"}, + {"16.4", "Where MySQL stores temporary files"}, + {"16.5", "Access denied error"}, + {"16.6", "How to run MySQL as a normal user"}, + {"16.7", "Problems with file permissions"}, + {"16.8", "File not found"}, + {"16.9", "Problems using DATE columns"}, + {"16.10", "Case sensitivity in searches"}, + {"16.11", "Problems with NULL values"}, + {"17", "Solving some common problems with MySQL"}, + {"17.1", "Database replication"}, + {"17.2", "Database backups"}, + {"18", "MySQL client tools and API's"}, + {"18.1", "MySQL C API"}, + {"18.2", "C API datatypes"}, + {"18.3", "C API function overview"}, + {"18.4", "C API function descriptions"}, + {"18.4.1", "mysql_affected_rows()"}, + {"18.4.2", "mysql_close()"}, + {"18.4.3", "mysql_connect()"}, + {"18.4.4", "mysql_create_db()"}, + {"18.4.5", "mysql_data_seek()"}, + {"18.4.6", "mysql_debug()"}, + {"18.4.7", "mysql_drop_db()"}, + {"18.4.8", "mysql_dump_debug_info()"}, + {"18.4.9", "mysql_eof()"}, + {"18.4.10", "mysql_errno()"}, + {"18.4.11", "mysql_error()"}, + {"18.4.12", "mysql_escape_string()"}, + {"18.4.13", "mysql_fetch_field()"}, + {"18.4.14", "mysql_fetch_fields()"}, + {"18.4.15", "mysql_fetch_field_direct()"}, + {"18.4.16", "mysql_fetch_lengths()"}, + {"18.4.17", "mysql_fetch_row()"}, + {"18.4.18", "mysql_field_seek()"}, + {"18.4.19", "mysql_field_tell()"}, + {"18.4.20", "mysql_free_result()"}, + {"18.4.21", "mysql_get_client_info()"}, + {"18.4.22", "mysql_get_host_info()"}, + {"18.4.23", "mysql_get_proto_info()"}, + {"18.4.24", "mysql_get_server_info()"}, + {"18.4.25", "mysql_info()"}, + {"18.4.26", "mysql_init()"}, + {"18.4.27", "mysql_insert_id()"}, + {"18.4.28", "mysql_kill()"}, + {"18.4.29", "mysql_list_dbs()"}, + {"18.4.30", "mysql_list_fields()"}, + {"18.4.31", "mysql_list_processes()"}, + {"18.4.32", "mysql_list_tables()"}, + {"18.4.33", "mysql_num_fields()"}, + {"18.4.34", "mysql_num_rows()"}, + {"18.4.35", "mysql_query()"}, + {"18.4.36", "mysql_real_connect()"}, + {"18.4.37", "mysql_real_query()"}, + {"18.4.38", "mysql_reload()"}, + {"18.4.39", "mysql_row_tell()"}, + {"18.4.40", "mysql_select_db()"}, + {"18.4.41", "mysql_shutdown()"}, + {"18.4.42", "mysql_stat()"}, + {"18.4.43", "mysql_store_result()"}, + {"18.4.44", "mysql_thread_id()"}, + {"18.4.45", "mysql_use_result()"}, + {"18.4.46", "Why is it that after mysql_query() returns success, mysql_store_result() sometimes returns NULL?"}, + {"18.4.47", "What results can I get from a query?"}, + {"18.4.48", "How can I get the unique ID for the last inserted row?"}, + {"18.4.49", "Problems linking with the C API"}, + {"18.4.50", "How to make a thread-safe client"}, + {"18.5", "MySQL Perl API's"}, + {"18.5.1", "DBI with DBD::mysql"}, + {"18.5.1.1", "The DBI interface"}, + {"18.5.1.2", "More DBI/DBD information"}, + {"18.6", "MySQL Java connectivity (JDBC)"}, + {"18.7", "MySQL PHP API's"}, + {"18.8", "MySQL C++ API's"}, + {"18.9", "MySQL Python API's"}, + {"18.10", "MySQL TCL API's"}, + {"19", "How MySQL compares to other databases"}, + {"19.1", "How MySQL compares to mSQL"}, + {"19.1.1", "How to convert mSQL tools for MySQL"}, + {"19.1.2", "How mSQL and MySQL client/server communications protocols differ"}, + {"19.1.3", "How mSQL 2.0 SQL syntax differs from MySQL"}, + {"19.2", "How MySQL compares to PostgreSQL"}, + {"A", "Some users of MySQL"}, + {"B", "Contributed programs"}, + {"C", "Contributors to MySQL"}, + {"D", "MySQL change history"}, + {"19.3", "Changes in release 3.22.x (Alpha version)"}, + {"19.3.1", "Changes in release 3.22.7"}, + {"19.3.2", "Changes in release 3.22.6"}, + {"19.3.3", "Changes in release 3.22.5"}, + {"19.3.4", "Changes in release 3.22.4"}, + {"19.3.5", "Changes in release 3.22.3"}, + {"19.3.6", "Changes in release 3.22.2"}, + {"19.3.7", "Changes in release 3.22.1"}, + {"19.3.8", "Changes in release 3.22.0"}, + {"19.4", "Changes in release 3.21.x"}, + {"19.4.1", "Changes in release 3.21.33"}, + {"19.4.2", "Changes in release 3.21.32"}, + {"19.4.3", "Changes in release 3.21.31"}, + {"19.4.4", "Changes in release 3.21.30"}, + {"19.4.5", "Changes in release 3.21.29"}, + {"19.4.6", "Changes in release 3.21.28"}, + {"19.4.7", "Changes in release 3.21.27"}, + {"19.4.8", "Changes in release 3.21.26"}, + {"19.4.9", "Changes in release 3.21.25"}, + {"19.4.10", "Changes in release 3.21.24"}, + {"19.4.11", "Changes in release 3.21.23"}, + {"19.4.12", "Changes in release 3.21.22"}, + {"19.4.13", "Changes in release 3.21.21a"}, + {"19.4.14", "Changes in release 3.21.21"}, + {"19.4.15", "Changes in release 3.21.20"}, + {"19.4.16", "Changes in release 3.21.19"}, + {"19.4.17", "Changes in release 3.21.18"}, + {"19.4.18", "Changes in release 3.21.17"}, + {"19.4.19", "Changes in release 3.21.16"}, + {"19.4.20", "Changes in release 3.21.15"}, + {"19.4.21", "Changes in release 3.21.14b"}, + {"19.4.22", "Changes in release 3.21.14a"}, + {"19.4.23", "Changes in release 3.21.13"}, + {"19.4.24", "Changes in release 3.21.12"}, + {"19.4.25", "Changes in release 3.21.11"}, + {"19.4.26", "Changes in release 3.21.10"}, + {"19.4.27", "Changes in release 3.21.9"}, + {"19.4.28", "Changes in release 3.21.8"}, + {"19.4.29", "Changes in release 3.21.7"}, + {"19.4.30", "Changes in release 3.21.6"}, + {"19.4.31", "Changes in release 3.21.5"}, + {"19.4.32", "Changes in release 3.21.4"}, + {"19.4.33", "Changes in release 3.21.3"}, + {"19.4.34", "Changes in release 3.21.2"}, + {"19.4.35", "Changes in release 3.21.0"}, + {"19.5", "Changes in release 3.20.x"}, + {"19.5.1", "Changes in release 3.20.18"}, + {"19.5.2", "Changes in release 3.20.17"}, + {"19.5.3", "Changes in release 3.20.16"}, + {"19.5.4", "Changes in release 3.20.15"}, + {"19.5.5", "Changes in release 3.20.14"}, + {"19.5.6", "Changes in release 3.20.13"}, + {"19.5.7", "Changes in release 3.20.11"}, + {"19.5.8", "Changes in release 3.20.10"}, + {"19.5.9", "Changes in release 3.20.9"}, + {"19.5.10", "Changes in release 3.20.8"}, + {"19.5.11", "Changes in release 3.20.7"}, + {"19.5.12", "Changes in release 3.20.6"}, + {"19.5.13", "Changes in release 3.20.3"}, + {"19.5.14", "Changes in release 3.20.0"}, + {"19.6", "Changes in release 3.19.x"}, + {"19.6.1", "Changes in release 3.19.5"}, + {"19.6.2", "Changes in release 3.19.4"}, + {"19.6.3", "Changes in release 3.19.3"}, + {"E", "Known errors and design deficiencies in MySQL"}, + {"F", "List of things we want to add to MySQL in the future (The TODO)"}, + {"19.7", "Things that must done in the real near future"}, + {"19.8", "Things that have to be done sometime"}, + {"19.9", "Some things we don't have any plans to do"}, + {"G", "Comments on porting to other systems"}, + {"19.10", "Debugging MySQL"}, + {"19.11", "Comments about RTS threads"}, + {"19.12", "What is the difference between different thread packages?"}, + {"H", "Description of MySQL regular expression syntax"}, + {"I", "What is Unireg?"}, + {"J", "The MySQL server license"}, + {"K", "The MySQL license for Microsoft operating systems"}, + {"*", "SQL command, type and function index"}, + {"*", "Concept Index"} +}; + +#define NQUERIES 5 +const char *query[NQUERIES]={ + "mysql information and manual", + "upgrading from previous version", + "column indexes", + "against about after more right the with/without", /* stopwords test */ + "mysql license and copyright" +}; + diff --git a/myisam/ft_update.c b/myisam/ft_update.c new file mode 100644 index 00000000000..cff5cb36baa --- /dev/null +++ b/myisam/ft_update.c @@ -0,0 +1,189 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* functions to work with full-text indices */ + +#include "ftdefs.h" + +/************************************************************** + This is to make ft-code to ignore keyseg.length at all * + and to index the whole VARCHAR/BLOB instead... */ +#undef set_if_smaller +#define set_if_smaller(A,B) /* no op */ +/**************************************************************/ + + +/* parses a document i.e. calls _mi_ft_parse for every keyseg */ +static FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, byte *keybuf, const byte *record) +{ + TREE *parsed=NULL; + MI_KEYSEG *keyseg; + byte *pos; + uint i; + + i=info->s->keyinfo[keynr].keysegs-FT_SEGS; + keyseg=info->s->keyinfo[keynr].seg; + while(i--) + { + uint len; + + keyseg--; + if (keyseg->null_bit && (record[keyseg->null_pos] & keyseg->null_bit)) + continue; /* NULL field */ + pos= (byte *)record+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH) + { + len=uint2korr(pos); + pos+=2; /* Skip VARCHAR length */ + set_if_smaller(len,keyseg->length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + len=_mi_calc_blob_length(keyseg->bit_start,pos); + memcpy_fixed(&pos,pos+keyseg->bit_start,sizeof(char*)); + set_if_smaller(len,keyseg->length); + } + else + len=keyseg->length; + + parsed=ft_parse(parsed, pos, len); + if(parsed==NULL) return NULL; + } + return ft_linearize(info, keynr, keybuf, parsed); +} + +static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf, + FT_WORD *wlist, my_off_t filepos) +{ + uint key_length; + + while(wlist->pos) + { + key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); + if (_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)) + return 1; + wlist++; + } + return 0; +} + +static int _mi_ft_erase(MI_INFO *info, uint keynr, byte *keybuf, FT_WORD *wlist, my_off_t filepos) +{ + uint key_length, err=0; + + while(wlist->pos) + { + key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); + if (_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)) + err=1; + wlist++; + } + return err; +} + +/* compares an appropriate parts of two WORD_KEY keys directly out of records */ +/* returns 1 if they are different */ + +#define THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT 1 +#define GEE_THEY_ARE_ABSOLUTELY_IDENTICAL 0 + +int _mi_ft_cmp(MI_INFO *info, uint keynr, const byte *rec1, const byte *rec2) +{ + MI_KEYSEG *keyseg; + byte *pos1, *pos2; + uint i; + + i=info->s->keyinfo[keynr].keysegs-FT_SEGS; + keyseg=info->s->keyinfo[keynr].seg; + while(i--) + { + uint len1, len2; + LINT_INIT(len1); LINT_INIT(len2); + keyseg--; + if (keyseg->null_bit) + { + if ( (rec1[keyseg->null_pos] ^ rec2[keyseg->null_pos]) + & keyseg->null_bit ) + return THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT; + if (rec1[keyseg->null_pos] & keyseg->null_bit ) + continue; /* NULL field */ + } + pos1= (byte *)rec1+keyseg->start; + pos2= (byte *)rec2+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH) + { + len1=uint2korr(pos1); + pos1+=2; /* Skip VARCHAR length */ + set_if_smaller(len1,keyseg->length); + len2=uint2korr(pos2); + pos2+=2; /* Skip VARCHAR length */ + set_if_smaller(len2,keyseg->length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + len1=_mi_calc_blob_length(keyseg->bit_start,pos1); + memcpy_fixed(&pos1,pos1+keyseg->bit_start,sizeof(char*)); + set_if_smaller(len1,keyseg->length); + len2=_mi_calc_blob_length(keyseg->bit_start,pos2); + memcpy_fixed(&pos2,pos2+keyseg->bit_start,sizeof(char*)); + set_if_smaller(len2,keyseg->length); + } + if ((len1 != len2) || memcmp(pos1, pos2, len1)) + return THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT; + } + return GEE_THEY_ARE_ABSOLUTELY_IDENTICAL; +} + +/* adds a document to the collection */ +int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, my_off_t pos) +{ + FT_WORD *wlist; + + if(!(wlist=_mi_ft_parserecord(info, keynr, keybuf, record))) return -1; + return _mi_ft_store(info,keynr,keybuf,wlist,pos); +} + +/* removes a document from the collection */ +int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, my_off_t pos) +{ + FT_WORD *wlist; + if(!(wlist=_mi_ft_parserecord(info, keynr, keybuf, record))) return -1; + return _mi_ft_erase(info,keynr,keybuf,wlist,pos); +} + +uint _ft_make_key(MI_INFO *info, uint keynr, byte *keybuf, FT_WORD *wptr, my_off_t filepos) +{ + byte buf[HA_FT_MAXLEN+16]; + +#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT + float weight=(float) ((filepos==HA_OFFSET_ERROR) ? 0 : wptr->weight); + mi_float4store(buf,weight); +#else +#error +#endif + +#ifdef EVAL_RUN + *(buf+HA_FT_WLEN)=wptr->cnt; + int2store(buf+HA_FT_WLEN+1,wptr->len); + memcpy(buf+HA_FT_WLEN+3,wptr->pos,wptr->len); +#else /* EVAL_RUN */ + int2store(buf+HA_FT_WLEN,wptr->len); + memcpy(buf+HA_FT_WLEN+2,wptr->pos,wptr->len); +#endif /* EVAL_RUN */ + return _mi_make_key(info,keynr,(uchar*) keybuf,buf,filepos); +} diff --git a/myisam/ftdefs.h b/myisam/ftdefs.h new file mode 100644 index 00000000000..ebf99e84d5a --- /dev/null +++ b/myisam/ftdefs.h @@ -0,0 +1,99 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* some definitions for full-text indices */ + +#include "fulltext.h" +#include <m_ctype.h> +#include <my_tree.h> + +#define MIN_WORD_LEN 4 + +#define HYPHEN_IS_DELIM +#define HYPHEN_IS_CONCAT /* not used for now */ + +#define COMPILE_STOPWORDS_IN + +/* Most of the formulae were shamelessly stolen from SMART distribution + ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z + NORM_PIVOT was taken from the article + A.Singhal, C.Buckley, M.Mitra, "Pivoted Document Length Normalization", + ACM SIGIR'96, 21-29, 1996 + */ + +#define LWS_FOR_QUERY LWS_TF +#define LWS_IN_USE LWS_LOG +#define PRENORM_IN_USE PRENORM_AVG +#define NORM_IN_USE NORM_PIVOT +#define GWS_IN_USE GWS_PROB +/*==============================================================*/ +#define LWS_TF (count) +#define LWS_BINARY (count>0) +#define LWS_SQUARE (count*count) +#define LWS_LOG (count?(log(count)+1):0) +/*--------------------------------------------------------------*/ +#define PRENORM_NONE (p->weight) +#define PRENORM_MAX (p->weight/docstat.max) +#define PRENORM_AUG (0.4+0.6*p->weight/docstat.max) +#define PRENORM_AVG (p->weight/docstat.sum*docstat.uniq) +#define PRENORM_AVGLOG ((1+log(p->weight))/(1+log(docstat.sum/docstat.uniq))) +/*--------------------------------------------------------------*/ +#define NORM_NONE (1) +#define NORM_SUM (docstat.nsum) +#define NORM_COS (sqrt(docstat.nsum2)) + +#ifdef EVAL_RUN +/* +extern ulong collstat; +#define PIVOT_STAT (docstat.uniq) +#define PIVOT_SLOPE (0.69) +#define PIVOT_PIVOT ((double)collstat/(info->state->records+1)) +#define NORM_PIVOT ((1-PIVOT_SLOPE)*PIVOT_PIVOT+PIVOT_SLOPE*docstat.uniq) +*/ +#endif /* EVAL_RUN */ + +#define PIVOT_VAL (0.0115) +#define NORM_PIVOT (1+PIVOT_VAL*docstat.uniq) +/*---------------------------------------------------------------*/ +#define GWS_NORM (1/sqrt(sum2)) +#define GWS_GFIDF (sum/doc_cnt) +/* Mysterious, but w/o (double) GWS_IDF performs better :-o */ +#define GWS_IDF log(aio->info->state->records/doc_cnt) +#define GWS_IDF1 log((double)aio->info->state->records/doc_cnt) +#define GWS_PROB log(((double)(aio->info->state->records-doc_cnt))/doc_cnt) +#define GWS_FREQ (1.0/doc_cnt) +#define GWS_SQUARED pow(log((double)aio->info->state->records/doc_cnt),2) +#define GWS_CUBIC pow(log((double)aio->info->state->records/doc_cnt),3) +#define GWS_ENTROPY (1-(suml/sum-log(sum))/log(aio->info->state->records)) +/*=================================================================*/ + +typedef struct st_ft_word { + byte * pos; + uint len; + double weight; +#ifdef EVAL_RUN + byte cnt; +#endif /* EVAL_RUN */ +} FT_WORD; + +int is_stopword(char *word, uint len); + +uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t); + +TREE * ft_parse(TREE *, byte *, int); +FT_WORD * ft_linearize(MI_INFO *, uint, byte *, TREE *); diff --git a/myisam/fulltext.h b/myisam/fulltext.h new file mode 100644 index 00000000000..3555abe1a1c --- /dev/null +++ b/myisam/fulltext.h @@ -0,0 +1,42 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* some definitions for full-text indices */ + +#include "myisamdef.h" +#include "ft_global.h" + +/* shoudn't be def'ed when linking with mysql */ +#undef EVAL_RUN + +#define HA_FT_MAXLEN 254 +#define HA_FT_WTYPE HA_KEYTYPE_FLOAT +#define HA_FT_WLEN 4 +#ifdef EVAL_RUN +#define FT_SEGS 3 +#else /* EVAL_RUN */ +#define FT_SEGS 2 +#endif /* EVAL_RUN */ + +extern const MI_KEYSEG ft_keysegs[FT_SEGS]; + +extern const char *ft_precompiled_stopwords[]; + +int _mi_ft_cmp(MI_INFO *, uint, const byte *, const byte *); +int _mi_ft_add(MI_INFO *, uint, byte *, const byte *, my_off_t); +int _mi_ft_del(MI_INFO *, uint, byte *, const byte *, my_off_t); diff --git a/myisam/make-ccc b/myisam/make-ccc new file mode 100755 index 00000000000..6d1303729db --- /dev/null +++ b/myisam/make-ccc @@ -0,0 +1,5 @@ +rm -f .deps/*.P +ccc -DMAP_TO_USE_RAID -I./../include -I../include -DDBUG_OFF -fast -O3 -c mi_cache.c mi_changed.c mi_checksum.c mi_close.c mi_create.c mi_dbug.c mi_delete.c mi_delete_all.c mi_delete_table.c mi_dynrec.c mi_extra.c mi_info.c mi_key.c mi_locking.c mi_log.c mi_open.c mi_packrec.c mi_page.c mi_panic.c mi_range.c mi_rename.c mi_rfirst.c mi_rkey.c mi_rlast.c mi_rnext.c mi_rnext_same.c mi_rprev.c mi_rrnd.c mi_rsame.c mi_rsamepos.c mi_scan.c mi_search.c mi_static.c mi_statrec.c mi_unique.c mi_update.c mi_write.c ft_update.c ft_search.o ft_stem.o ft_stopwords.c ft_parser.c +make sort.o mi_check.o +rm libmyisam.a +ar -cr libmyisam.a mi_cache.o sort.o mi_check.o diff --git a/myisam/mi_cache.c b/myisam/mi_cache.c new file mode 100644 index 00000000000..f2f7af8eac3 --- /dev/null +++ b/myisam/mi_cache.c @@ -0,0 +1,101 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Functions for read record cacheing with myisam */ +/* Used instead of my_b_read() to allow for no-cacheed seeks */ + +#include "myisamdef.h" +#define READING_NEXT 1 +#define READING_HEADER 2 + + /* Copy block from cache if it`s in it. If re_read_if_possibly is */ + /* set read to cache (if after current file-position) else read to */ + /* buff */ + +int _mi_read_cache(IO_CACHE *info, byte *buff, my_off_t pos, uint length, + int flag) +{ + uint read_length,in_buff_length; + my_off_t offset; + char *in_buff_pos; + DBUG_ENTER("_mi_read_cache"); + + if (pos < info->pos_in_file) + { + read_length=length; + if ((my_off_t) read_length > (my_off_t) (info->pos_in_file-pos)) + read_length=(uint) (info->pos_in_file-pos); + info->seek_not_done=1; + VOID(my_seek(info->file,pos,MY_SEEK_SET,MYF(0))); + if (my_read(info->file,buff,read_length,MYF(MY_NABP))) + DBUG_RETURN(1); + if (!(length-=read_length)) + DBUG_RETURN(0); + pos+=read_length; + buff+=read_length; + } + if ((offset= (my_off_t) (pos - info->pos_in_file)) < + (my_off_t) (info->rc_end - info->rc_request_pos)) + { + in_buff_pos=info->rc_request_pos+(uint) offset; + in_buff_length= min(length,(uint) (info->rc_end-in_buff_pos)); + memcpy(buff,info->rc_request_pos+(uint) offset,(size_t) in_buff_length); + if (!(length-=in_buff_length)) + DBUG_RETURN(0); + pos+=in_buff_length; + buff+=in_buff_length; + } + else + in_buff_length=0; + if (flag & READING_NEXT) + { + if (pos != ((info)->pos_in_file + + (uint) ((info)->rc_end - (info)->rc_request_pos))) + { + info->pos_in_file=pos; /* Force start here */ + info->rc_pos=info->rc_end=info->rc_request_pos; /* Everything used */ + info->seek_not_done=1; + } + else + info->rc_pos=info->rc_end; /* All block used */ + if (!(*info->read_function)(info,buff,length)) + DBUG_RETURN(0); + if (!(flag & READING_HEADER) || info->error == -1 || + (uint) info->error+in_buff_length < 3) + { + if (!my_errno) + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + bzero(buff+info->error,MI_BLOCK_INFO_HEADER_LENGTH - in_buff_length - + (uint) info->error); + DBUG_RETURN(0); + } + info->seek_not_done=1; + VOID(my_seek(info->file,pos,MY_SEEK_SET,MYF(0))); + if ((read_length=my_read(info->file,buff,length,MYF(0))) == length) + DBUG_RETURN(0); + if (!(flag & READING_HEADER) || (int) read_length == -1 || + read_length+in_buff_length < 3) + { + if (!my_errno) + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + bzero(buff+read_length,MI_BLOCK_INFO_HEADER_LENGTH - in_buff_length - + read_length); + DBUG_RETURN(0); +} /* _mi_read_cache */ diff --git a/myisam/mi_changed.c b/myisam/mi_changed.c new file mode 100644 index 00000000000..bd6b14b0c6c --- /dev/null +++ b/myisam/mi_changed.c @@ -0,0 +1,34 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Check if somebody has changed table since last check. */ + +#include "myisamdef.h" + + /* Return 0 if table isn't changed */ + +int mi_is_changed(MI_INFO *info) +{ + int result; + DBUG_ENTER("mi_is_changed"); + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(-1); + VOID(_mi_writeinfo(info,0)); + result=(int) info->data_changed; + info->data_changed=0; + DBUG_PRINT("exit",("result: %d",result)); + DBUG_RETURN(result); +} diff --git a/myisam/mi_check.c b/myisam/mi_check.c new file mode 100644 index 00000000000..5b5583b136e --- /dev/null +++ b/myisam/mi_check.c @@ -0,0 +1,2886 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Descript, check and repair of ISAM tables */ + +#include "myisamdef.h" + +#include <m_ctype.h> +#include <stdarg.h> +#include <getopt.h> +#include <assert.h> +#ifdef HAVE_SYS_VADVICE_H +#include <sys/vadvise.h> +#endif +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif + +#ifndef USE_RAID +#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G) +#define my_raid_delete(A,B,C) my_delete(A,B) +#endif + + /* Functions defined in this file */ + +static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr); +static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, ha_rows *keys, + ha_checksum *key_checksum, uint level); +static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo); +static ha_checksum calc_checksum(ha_rows count); +static int writekeys(MI_INFO *info,byte *buff,my_off_t filepos); +static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo, + my_off_t pagepos, File new_file); +static int sort_key_read(SORT_INFO *sort_info,void *key); +static int sort_get_next_record(SORT_INFO *sort_info); +static int sort_key_cmp(SORT_INFO *sort_info, const void *a,const void *b); +static int sort_key_write(SORT_INFO *sort_info, const void *a); +static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo, + uchar *key); +static int sort_insert_key(MI_CHECK *param, reg1 SORT_KEY_BLOCKS *key_block, + uchar *key, my_off_t prev_block); +static int sort_delete_record(MI_CHECK *param); +static int flush_pending_blocks(MI_CHECK *param); +static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks, + uint buffer_length); +static void update_key_parts(MI_KEYDEF *keyinfo, + ulong *rec_per_key_part, + ulonglong *unique, + ulonglong records); +static ha_checksum mi_byte_checksum(const byte *buf, uint length); + + +#ifdef __WIN__ +static double ulonglong2double(ulonglong value) +{ + longlong nr=(longlong) value; + if (nr >= 0) + return (double) nr; + return (18446744073709551616.0 + (double) nr); +} + +#if SIZEOF_OFF_T > 4 +#define my_off_t2double(A) ulonglong2double(A) +#else +#define my_off_t2double(A) ((double) (A)) +#endif /* SIZEOF_OFF_T > 4 */ +#endif + +void myisamchk_init(MI_CHECK *param) +{ + bzero((gptr) param,sizeof(*param)); + param->opt_follow_links=1; + param->keys_in_use= ~(ulonglong) 0; + param->search_after_block=HA_OFFSET_ERROR; + param->auto_increment_value= 0; + param->use_buffers=USE_BUFFER_INIT; + param->read_buffer_length=READ_BUFFER_INIT; + param->write_buffer_length=READ_BUFFER_INIT; + param->sort_buffer_length=SORT_BUFFER_INIT; + param->sort_key_blocks=BUFFERS_WHEN_SORTING; + param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; + param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL); + param->sort_info.param=param; +} + + /* Check delete links */ + +int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag) +{ + reg2 ha_rows i; + uint j,delete_link_length; + my_off_t empty,next_link,old_link; + char buff[22],buff2[22]; + DBUG_ENTER("chk_del"); + + if (!(test_flag & T_SILENT)) + puts("- check key delete-chain"); + + LINT_INIT(old_link); + param->record_checksum=0; + param->key_file_blocks=info->s->base.keystart; + for (j=0 ; j < info->s->state.header.max_block_size ; j++) + if (check_k_link(param,info,j)) + goto wrong; + delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : + info->s->rec_reflength+1); + + if (!(test_flag & T_SILENT)) + puts("- check record delete-chain"); + + next_link=info->s->state.dellink; + if (info->state->del == 0) + { + if (test_flag & T_VERBOSE) + { + puts("No recordlinks"); + } + } + else + { + if (test_flag & T_VERBOSE) + printf("Recordlinks: "); + empty=0; + for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--) + { + if (test_flag & T_VERBOSE) + printf(" %9s",llstr(next_link,buff)); + if (next_link >= info->state->data_file_length) + goto wrong; + if (my_pread(info->dfile,(char*) buff,delete_link_length, + next_link,MYF(MY_NABP))) + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Can't read delete-link at filepos: %s", + llstr(next_link,buff)); + DBUG_RETURN(1); + } + if (*buff != '\0') + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Record at pos: %s is not remove-marked", + llstr(next_link,buff)); + goto wrong; + } + if (info->s->options & HA_OPTION_PACK_RECORD) + { + my_off_t prev_link=mi_sizekorr(buff+12); + if (empty && prev_link != old_link) + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2)); + goto wrong; + } + old_link=next_link; + next_link=mi_sizekorr(buff+4); + empty+=mi_uint3korr(buff+1); + } + else + { + param->record_checksum+=(ha_checksum) next_link; + next_link=_mi_rec_pos(info->s,(uchar*) buff+1); + empty+=info->s->base.pack_reclength; + } + } + if (empty != info->state->empty) + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_warning(param, + "Not used space is supposed to be: %s but is: %s", + llstr(info->state->empty,buff), + llstr(empty,buff2)); + info->state->empty=empty; + } + if (i != 0 || next_link != HA_OFFSET_ERROR) + goto wrong; + + if (test_flag & T_VERBOSE) puts("\n"); + } + DBUG_RETURN(0); +wrong: + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"record delete-link-chain corrupted"); + DBUG_RETURN(1); +} /* chk_del */ + + + /* Check delete links in index file */ + +static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr) +{ + my_off_t next_link; + uint block_size=(nr+1)*MI_KEY_BLOCK_LENGTH; + ha_rows records; + char llbuff[21],*buff; + DBUG_ENTER("check_k_link"); + + if (param->testflag & T_VERBOSE) + printf("block_size %4d:",block_size); + + next_link=info->s->state.key_del[nr]; + records= (ha_rows) (info->state->key_file_length / block_size); + while (next_link != HA_OFFSET_ERROR && records > 0) + { + if (param->testflag & T_VERBOSE) + printf("%16s",llstr(next_link,llbuff)); + if (next_link > info->state->key_file_length || + next_link & (info->s->blocksize-1)) + DBUG_RETURN(1); + if (!(buff=key_cache_read(info->s->kfile, next_link, info->buff, + myisam_block_size, block_size, 1))) + DBUG_RETURN(1); + next_link=mi_sizekorr(buff); + records--; + param->key_file_blocks+=block_size; + } + if (param->testflag & T_VERBOSE) + { + if (next_link != HA_OFFSET_ERROR) + printf("%16s\n",llstr(next_link,llbuff)); + else + puts(""); + } + DBUG_RETURN (next_link != HA_OFFSET_ERROR); +} /* check_k_link */ + + + /* Kontrollerar storleken p} filerna */ + +int chk_size(MI_CHECK *param, register MI_INFO *info) +{ + int error=0; + register my_off_t skr,size; + char buff[22],buff2[22]; + DBUG_ENTER("chk_size"); + + if (!(param->testflag & T_SILENT)) puts("- check file-size"); + + flush_key_blocks(info->s->kfile, FLUSH_FORCE_WRITE); /* If called externally */ + + size=my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0)); + if ((skr=(my_off_t) info->state->key_file_length) != size) + { + if (skr > size) + { + error=1; + mi_check_print_error(param, + "Size of indexfile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + else + mi_check_print_warning(param, + "Size of indexfile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + if (!(param->testflag & T_VERY_SILENT) && + ! (info->s->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(info->state->key_file_length) > + ulonglong2double(info->s->base.margin_key_file_length)*0.9) + mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used", + llstr(info->state->key_file_length,buff), + llstr(info->s->base.max_key_file_length-1,buff)); + + size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)); + skr=(my_off_t) info->state->data_file_length; + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + skr+= MEMMAP_EXTRA_MARGIN; +#ifdef USE_RELOC + if (info->data_file_type == STATIC_RECORD && + skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length) + skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length; +#endif + if (skr != size) + { + info->state->data_file_length=size; /* Skipp other errors */ + if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN) + { + error=1; + mi_check_print_error(param,"Size of datafile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + else + { + mi_check_print_warning(param, + "Size of datafile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + } + if (!(param->testflag & T_VERY_SILENT) && + !(info->s->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(info->state->data_file_length) > + (ulonglong2double(info->s->base.max_data_file_length)*0.9)) + mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used", + llstr(info->state->data_file_length,buff), + llstr(info->s->base.max_data_file_length-1,buff2)); + DBUG_RETURN(error); +} /* chk_size */ + + + /* Check keys */ + +int chk_key(MI_CHECK *param, register MI_INFO *info) +{ + uint key,found_keys=0,full_text_keys=0; + ha_rows keys; + ha_checksum old_record_checksum,init_checksum; + my_off_t all_keydata,all_totaldata,key_totlength,length; + ulong *rec_per_key_part; + MYISAM_SHARE *share=info->s; + MI_KEYDEF *keyinfo; + char buff[22],buff2[22]; + DBUG_ENTER("chk_key"); + + if (!(param->testflag & T_SILENT)) puts("- check index reference"); + + all_keydata=all_totaldata=key_totlength=0; + old_record_checksum=0; + init_checksum=param->record_checksum; + if (!(share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD))) + old_record_checksum=calc_checksum(info->state->records+info->state->del-1)* + share->base.pack_reclength; + rec_per_key_part= param->rec_per_key_part; + for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ; + rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++) + { + param->key_crc[key]=0; + if (!(((ulonglong) 1 << key) & share->state.key_map)) + continue; + found_keys++; + + param->record_checksum=init_checksum; + bzero((char*) ¶m->unique_count,sizeof(param->unique_count)); + if ((!(param->testflag & T_SILENT))) + printf ("- check data record references index: %d\n",key+1); + if (share->state.key_root[key] == HA_OFFSET_ERROR && + info->state->records == 0) + continue; + if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],info->buff, + 0)) + { + mi_check_print_error(param,"Can't read indexpage from filepos: %s", + llstr(share->state.key_root[key],buff)); + DBUG_RETURN(-1); + } + param->key_file_blocks+=keyinfo->block_length; + keys=0; + param->keydata=param->totaldata=0; + param->key_blocks=0; + param->max_level=0; + if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff, + &keys, param->key_crc+key,1)) + DBUG_RETURN(-1); + if(!(keyinfo->flag & HA_FULLTEXT)) + { + if (keys != info->state->records) + { + mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff), + llstr(info->state->records,buff2)); + DBUG_RETURN(-1); + } + if (found_keys - full_text_keys == 1 && + ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) || + (param->testflag & T_DONT_CHECK_CHECKSUM))) + old_record_checksum=param->record_checksum; + else if (old_record_checksum != param->record_checksum) + { + if (key) + mi_check_print_error(param,"Key %u doesn't point at same records that key 1", + key+1); + else + mi_check_print_error(param,"Key 1 doesn't point at all records"); + DBUG_RETURN(-1); + } + } + else + full_text_keys++; + /* Check that auto_increment key is bigger than max key value */ + if ((uint) share->base.auto_key -1 == key) + { + ulonglong save_auto_value=info->s->state.auto_increment; + info->s->state.auto_increment=0; + info->lastinx=key; + _mi_read_key_record(info, 0L, info->rec_buff); + update_auto_increment(info, info->rec_buff); + if (info->s->state.auto_increment > save_auto_value) + { + mi_check_print_warning(param, + "Auto-increment value: %s is smaller than max used value: %s", + llstr(save_auto_value,buff2), + llstr(info->s->state.auto_increment, buff)); + } + if (param->testflag & T_AUTO_INC) + { + set_if_bigger(info->s->state.auto_increment, + param->auto_increment_value); + } + else + info->s->state.auto_increment=save_auto_value; + } + + length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2; + if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L) + printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n", + key+1, + (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)), + (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/ + my_off_t2double(length)), + param->max_level); + all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length; + + if (param->testflag & T_STATISTICS) + update_key_parts(keyinfo, rec_per_key_part, param->unique_count, + (ulonglong) info->state->records); + } + if (param->testflag & T_INFO) + { + if (all_totaldata != 0L && found_keys > 0) + printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n", + (int) (my_off_t2double(all_keydata)*100.0/ + my_off_t2double(all_totaldata)), + (int) ((my_off_t2double(key_totlength) - + my_off_t2double(all_keydata))*100.0/ + my_off_t2double(key_totlength))); + else if (all_totaldata != 0L && share->state.key_map) + puts(""); + } + if (param->key_file_blocks != info->state->key_file_length && + param->keys_in_use != ~(ulonglong) 0) + mi_check_print_warning(param, "Some data are unreferenced in keyfile"); + if (found_keys != full_text_keys) + param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */ + else + param->record_checksum=0; + DBUG_RETURN(0); +} /* chk_key */ + + + /* Check if index is ok */ + +static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, ha_rows *keys, + ha_checksum *key_checksum, uint level) +{ + int flag; + uint used_length,comp_flag,nod_flag,key_length,not_used; + uchar key[MI_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos; + my_off_t next_page,record; + char llbuff[22],llbuff2[22]; + DBUG_ENTER("chk_index"); + DBUG_DUMP("buff",(byte*) buff,mi_getint(buff)); + + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not Enough memory"); + DBUG_RETURN(-1); + } + + if (keyinfo->flag & HA_NOSAME) + comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */ + else + comp_flag=SEARCH_SAME; /* Keys in positionorder */ + nod_flag=mi_test_if_nod(buff); + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + + param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */ + param->key_blocks++; + if (level > param->max_level) + param->max_level=level; + + if (used_length > keyinfo->block_length) + { + mi_check_print_error(param,"Wrong pageinfo at page: %s", llstr(page,llbuff)); + goto err; + } + for ( ;; ) + { + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + if (next_page > info->state->key_file_length || + (nod_flag && (next_page & (info->s->blocksize -1)))) + { + my_off_t max_length=my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0)); + mi_check_print_error(param,"Wrong pagepointer: %s at page: %s", + llstr(next_page,llbuff),llstr(page,llbuff2)); + + if (next_page+info->s->blocksize > max_length) + goto err; + info->state->key_file_length=(max_length & + ~ (my_off_t) (info->s->blocksize-1)); + } + if (!_mi_fetch_keypage(info,keyinfo,next_page,temp_buff,0)) + { + mi_check_print_error(param,"Can't read key from filepos: %s",llstr(next_page,llbuff)); + goto err; + } + param->key_file_blocks+=keyinfo->block_length; + if (chk_index(param,info,keyinfo,next_page,temp_buff,keys,key_checksum, + level+1)) + goto err; + } + old_keypos=keypos; + if (keypos >= endpos || + (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0) + break; + if (keypos > endpos) + { + mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff)); + goto err; + } + if ((*keys)++ && + (flag=_mi_key_cmp(keyinfo->seg,info->lastkey,key,key_length, + comp_flag, ¬_used)) >=0) + { + DBUG_DUMP("old",(byte*) info->lastkey, info->lastkey_length); + DBUG_DUMP("new",(byte*) key, key_length); + DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos)); + + if (comp_flag & SEARCH_FIND && flag == 0) + mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff)); + else + mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff)); + goto err; + } + if (param->testflag & T_STATISTICS) + { + if (*keys == 1L) /* first_key */ + param->unique_count[keyinfo->keysegs]++; + else + { + uint diff; + _mi_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,SEARCH_FIND, + &diff); + param->unique_count[diff-1]++; + } + } + (*key_checksum)+= mi_byte_checksum((byte*) key, + key_length- info->s->rec_reflength); + memcpy((char*) info->lastkey,(char*) key,key_length); + info->lastkey_length=key_length; + record= _mi_dpos(info,0,key+key_length); + if (record >= info->state->data_file_length) + { +#ifndef DBUG_OFF + char llbuff3[22]; +#endif + mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff)); + DBUG_PRINT("test",("page: %s record: %s filelength: %s", + llstr(page,llbuff),llstr(record,llbuff2), + llstr(info->state->data_file_length,llbuff3))); + DBUG_DUMP("key",(byte*) info->lastkey,key_length); + DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos)); + goto err; + } + param->record_checksum+=(ha_checksum) record; + } + if (keypos != endpos) + { + mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d", + llstr(page,llbuff), used_length, (keypos - buff)); + goto err; + } + my_afree((byte*) temp_buff); + DBUG_RETURN(0); + err: + my_afree((byte*) temp_buff); + DBUG_RETURN(1); +} /* chk_index */ + + + /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */ + +static ha_checksum calc_checksum(ha_rows count) +{ + ulonglong sum,a,b; + DBUG_ENTER("calc_checksum"); + + sum=0; + a=count; b=count+1; + if (a & 1) + b>>=1; + else + a>>=1; + while (b) + { + if (b & 1) + sum+=a; + a<<=1; b>>=1; + } + DBUG_PRINT("exit",("sum: %lx",(ulong) sum)); + DBUG_RETURN((ha_checksum) sum); +} /* calc_checksum */ + + + /* Calc length of key in normal isam */ + +static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo) +{ + uint length; + MI_KEYSEG *keyseg; + DBUG_ENTER("isam_key_length"); + + length= info->s->rec_reflength; + for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++) + length+= keyseg->length; + + DBUG_PRINT("exit",("length: %d",length)); + DBUG_RETURN(length); +} /* key_length */ + + + /* Check that record-link is ok */ + +int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) +{ + int error,got_error,flag; + uint key,left_length,b_type,field; + ha_rows records,del_blocks; + my_off_t used,empty,pos,splits,start_recpos, + del_length,link_used,start_block; + byte *record,*to; + char llbuff[22],llbuff2[22],llbuff3[22]; + ha_checksum intern_record_checksum; + ha_checksum key_checksum[MI_MAX_POSSIBLE_KEY]; + my_bool static_row_size; + MI_KEYDEF *keyinfo; + MI_BLOCK_INFO block_info; + DBUG_ENTER("chk_data_link"); + + if (!(param->testflag & T_SILENT)) + { + if (extend) + puts("- check records and index references"); + else + puts("- check record links"); + } + + if (!(record= (byte*) my_alloca(info->s->base.pack_reclength))) + { + mi_check_print_error(param,"Not Enough memory"); + DBUG_RETURN(-1); + } + records=del_blocks=0; + used=link_used=splits=del_length=0; + intern_record_checksum=param->glob_crc=0; + LINT_INIT(left_length); LINT_INIT(start_recpos); LINT_INIT(to); + got_error=error=0; + empty=pos=info->s->pack.header_length; + + /* Check how to calculate checksum of rows */ + static_row_size=1; + if (info->s->data_file_type == COMPRESSED_RECORD) + { + for (field=0 ; field < info->s->base.fields ; field++) + { + if (info->s->rec[field].base_type == FIELD_BLOB || + info->s->rec[field].base_type == FIELD_VARCHAR) + { + static_row_size=0; + break; + } + } + } + + bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0])); + while (pos < info->state->data_file_length) + { + switch (info->s->data_file_type) { + case STATIC_RECORD: + if (my_b_read(¶m->read_cache,(byte*) record, + info->s->base.pack_reclength)) + goto err; + start_recpos=pos; + pos+=info->s->base.pack_reclength; + splits++; + if (*record == '\0') + { + del_blocks++; + del_length+=info->s->base.pack_reclength; + continue; /* Record removed */ + } + param->glob_crc+= mi_static_checksum(info,record); + used+=info->s->base.pack_reclength; + break; + case DYNAMIC_RECORD: + flag=block_info.second_read=0; + block_info.next_filepos=pos; + do + { + if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header, + (start_block=block_info.next_filepos), + sizeof(block_info.header),test(! flag) | 2)) + goto err; + if (start_block & (MI_DYN_ALIGN_SIZE-1)) + { + mi_check_print_error(param,"Wrong aligned block at %s",llstr(start_block,llbuff)); + goto err2; + } + b_type=_mi_get_block_info(&block_info,-1,start_block); + if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & BLOCK_SYNC_ERROR) + { + if (flag) + { + mi_check_print_error(param,"Unexpected byte: %d at link: %s", + (int) block_info.header[0], + llstr(start_block,llbuff)); + goto err2; + } + pos=block_info.filepos+block_info.block_len; + goto next; + } + if (b_type & BLOCK_DELETED) + { + if (block_info.block_len < info->s->base.min_block_length) + { + mi_check_print_error(param,"Deleted block with impossible length %lu at %s", + block_info.block_len,llstr(pos,llbuff)); + goto err2; + } + if ((block_info.next_filepos != HA_OFFSET_ERROR && + block_info.next_filepos >= info->state->data_file_length) || + (block_info.prev_filepos != HA_OFFSET_ERROR && + block_info.prev_filepos >= info->state->data_file_length)) + { + mi_check_print_error(param,"Delete link points outside datafile at %s", + llstr(pos,llbuff)); + goto err2; + } + del_blocks++; + del_length+=block_info.block_len; + pos=block_info.filepos+block_info.block_len; + splits++; + goto next; + } + mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s", + block_info.header[0],block_info.header[1], + block_info.header[2], + llstr(start_block,llbuff)); + goto err2; + } + if (info->state->data_file_length < block_info.filepos+ + block_info.block_len) + { + mi_check_print_error(param,"Recordlink that points outside datafile at %s", + llstr(pos,llbuff)); + got_error=1; + break; + } + splits++; + if (!flag++) /* First block */ + { + start_recpos=pos; + pos=block_info.filepos+block_info.block_len; + if (block_info.rec_len > (uint) info->s->base.max_pack_length) + { + mi_check_print_error(param,"Found too long record (%d) at %s", + block_info.rec_len, + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (info->s->base.blobs) + { + if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + { + mi_check_print_error(param,"Not enough memory for blob at %s", + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + } + else + to= info->rec_buff; + left_length=block_info.rec_len; + } + if (left_length < block_info.data_len) + { + mi_check_print_error(param,"Found too long record at %s", + llstr(start_recpos,llbuff)); + got_error=1; break; + } + if (_mi_read_cache(¶m->read_cache,(byte*) to,block_info.filepos, + (uint) block_info.data_len, test(flag == 1))) + goto err; + to+=block_info.data_len; + link_used+= block_info.filepos-start_block; + used+= block_info.filepos - start_block + block_info.data_len; + empty+=block_info.block_len-block_info.data_len; + left_length-=block_info.data_len; + if (left_length) + { + if (b_type & BLOCK_LAST) + { + mi_check_print_error(param,"Record link to short for record at %s", + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (info->state->data_file_length < block_info.next_filepos) + { + mi_check_print_error(param,"Found next-recordlink that points outside datafile at %s", + llstr(block_info.filepos,llbuff)); + got_error=1; + break; + } + } + } while (left_length); + if (! got_error) + { + if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) == + MY_FILE_ERROR) + { + mi_check_print_error(param,"Found wrong record at %s", llstr(start_recpos,llbuff)); + got_error=1; + } + else + { + info->checksum=mi_checksum(info,record); + if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE)) + { + if (_mi_rec_check(info,record)) + { + mi_check_print_error(param,"Found wrong packed record at %s", + llstr(start_recpos,llbuff)); + got_error=1; + } + } + if (!got_error) + param->glob_crc+= info->checksum; + } + } + else if (!flag) + pos=block_info.filepos+block_info.block_len; + break; + case COMPRESSED_RECORD: + if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header, pos, + info->s->pack.ref_length, 1)) + goto err; + start_recpos=pos; + splits++; + VOID(_mi_pack_get_block_info(info,&block_info, -1, start_recpos, NullS)); + pos=block_info.filepos+block_info.rec_len; + if (block_info.rec_len < (uint) info->s->min_pack_length || + block_info.rec_len > (uint) info->s->max_pack_length) + { + mi_check_print_error(param,"Found block with wrong recordlength: %d at %s", + block_info.rec_len, llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (_mi_read_cache(¶m->read_cache,(byte*) info->rec_buff, + block_info.filepos, block_info.rec_len,1)) + goto err; + if (_mi_pack_rec_unpack(info,record,info->rec_buff,block_info.rec_len)) + { + mi_check_print_error(param,"Found wrong record at %s", llstr(start_recpos,llbuff)); + got_error=1; + } + if (static_row_size) + param->glob_crc+= mi_static_checksum(info,record); + else + param->glob_crc+= mi_checksum(info,record); + link_used+= (block_info.filepos - start_recpos); + used+= (pos-start_recpos); + } /* switch */ + if (! got_error) + { + intern_record_checksum+=(ha_checksum) start_recpos; + records++; + if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0) + { + printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout)); + } + + /* Check if keys match the record */ + + for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys; + key++,keyinfo++) + { + if ((((ulonglong) 1 << key) & info->s->state.key_map)) + { + if(!(keyinfo->flag & HA_FULLTEXT)) + { + uint key_length=_mi_make_key(info,key,info->lastkey,record, + start_recpos); + if (extend) + { + /* We don't need to lock the key tree here as we don't allow + concurrent threads when running myisamchk + */ + if (_mi_search(info,keyinfo,info->lastkey,key_length, + SEARCH_SAME, info->s->state.key_root[key])) + { + mi_check_print_error(param,"Record at: %10s Can't find key for index: %2d", + llstr(start_recpos,llbuff),key+1); + if (error++ > MAXERR || !(param->testflag & T_VERBOSE)) + goto err2; + } + } + else + key_checksum[key]+=mi_byte_checksum((byte*) info->lastkey, + key_length); + } + } + } + } + else + { + got_error=0; + if (error++ > MAXERR || !(param->testflag & T_VERBOSE)) + goto err2; + } + next:; /* Next record */ + } + if (param->testflag & T_WRITE_LOOP) + { + VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); + } + if (records != info->state->records) + { + mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s", + llstr(records,llbuff), llstr(info->state->records,llbuff2)); + error=1; + } + else if (param->record_checksum && + param->record_checksum != intern_record_checksum) + { + mi_check_print_error(param, + "Keypointers and record positions doesn't match"); + error=1; + } + else if (param->glob_crc != info->s->state.checksum && + (info->s->options & + (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))) + { + mi_check_print_warning(param, + "Record checksum is not the same as checksum stored in the index file\n"); + error=1; + } + else if (!extend) + { + for (key=0 ; key < info->s->base.keys; key++) + { + if (key_checksum[key] != param->key_crc[key] && + !(info->s->keyinfo[key].flag & HA_FULLTEXT)) + { + mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records", + key+1); + error=1; + } + } + } + + if (used+empty+del_length != info->state->data_file_length) + { + mi_check_print_warning(param, + "Found %s record-data and %s unused data and %s deleted-data", + llstr(used,llbuff),llstr(empty,llbuff2), + llstr(del_length,llbuff3)); + mi_check_print_warning(param, + "Total %s, Should be: %s", + llstr((used+empty+del_length),llbuff), + llstr(info->state->data_file_length,llbuff2)); + } + if (del_blocks != info->state->del) + { + mi_check_print_warning(param, + "Found %10s deleted blocks Should be: %s", + llstr(del_blocks,llbuff), + llstr(info->state->del,llbuff2)); + } + if (splits != info->s->state.split) + { + mi_check_print_warning(param, + "Found %10s parts Should be: %s parts", + llstr(splits,llbuff), + llstr(info->s->state.split,llbuff2)); + } + if (param->testflag & T_INFO) + { + if (param->warning_printed || param->error_printed) + puts(""); + if (used != 0 && ! param->error_printed) + { + printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n", + llstr(records,llbuff), (long)((used-link_used)/records), + (info->s->base.blobs ? 0.0 : + (ulonglong2double((ulonglong) info->s->base.reclength*records)- + my_off_t2double(used))/ + ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0)); + printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n", + (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0), + (!records ? 100 : (int) (ulonglong2double(del_length+empty)/ + my_off_t2double(used)*100.0)), + ulonglong2double(splits - del_blocks) / records); + } + printf("Record blocks:%12s Delete blocks:%10s\n", + llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2)); + printf("Record data: %12s Deleted data: %10s\n", + llstr(used-link_used,llbuff),llstr(del_length,llbuff2)); + printf("Lost space: %12s Linkdata: %10s\n", + llstr(empty,llbuff),llstr(link_used,llbuff2)); + } + my_afree((gptr) record); + DBUG_RETURN (error); + err: + mi_check_print_error(param,"got error: %d when reading datafile",my_errno); + err2: + my_afree((gptr) record); + DBUG_RETURN(1); +} /* chk_data_link */ + + + /* Recover old table by reading each record and writing all keys */ + /* Save new datafile-name in temp_filename */ + +int mi_repair(MI_CHECK *param, register MI_INFO *info, + my_string name, int rep_quick) +{ + int error,got_error; + uint i; + ha_rows start_records,new_header_length; + my_off_t del; + File new_file; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("mi_repair"); + + start_records=info->state->records; + new_header_length=(param->testflag & T_UNPACK) ? 0L : share->pack.header_length; + got_error=1; + new_file= -1; + if (!(param->testflag & T_SILENT)) + { + printf("- recovering MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + } + + if (!param->using_global_keycache) + VOID(init_key_cache(param->use_buffers,NEAD_MEM)); + + if (init_io_cache(¶m->read_cache,info->dfile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1,MYF(MY_WME))) + goto err; + if (!rep_quick) + if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length, + WRITE_CACHE, new_header_length, 1, + MYF(MY_WME | MY_WAIT_IF_FULL))) + goto err; + info->opt_flag|=WRITE_CACHE_USED; + sort_info->start_recpos=0; + sort_info->buff=0; sort_info->buff_length=0; + if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, + MYF(0)))) + { + mi_check_print_error(param,"Not Enough memory for extra record"); + goto err; + } + + if (!rep_quick) + { + if ((new_file=my_raid_create(fn_format(param->temp_filename,name,"", + DATA_TMP_EXT, + 2+4), + 0,param->tmpfile_createflag, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (filecopy(param,new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + info->s->state.dellink= HA_OFFSET_ERROR; + info->rec_cache.file=new_file; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + } + sort_info->info=info; + sort_info->pos=sort_info->max_pos=share->pack.header_length; + sort_info->filepos=new_header_length; + param->read_cache.end_of_file=sort_info->filelength= + my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)); + sort_info->dupp=0; + sort_info->fix_datafile= (my_bool) (! rep_quick); + sort_info->max_records= ~(ha_rows) 0; + if ((sort_info->new_data_file_type=share->data_file_type) == + COMPRESSED_RECORD && param->testflag & T_UNPACK) + { + if (share->options & HA_OPTION_PACK_RECORD) + sort_info->new_data_file_type = DYNAMIC_RECORD; + else + sort_info->new_data_file_type = STATIC_RECORD; + } + + del=info->state->del; + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + if (sort_info->new_data_file_type != COMPRESSED_RECORD && !rep_quick) + share->state.checksum=0; + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + for (i=0 ; i < info->s->base.keys ; i++) + share->state.key_root[i]= HA_OFFSET_ERROR; + for (i=0 ; i < share->state.header.max_block_size ; i++) + share->state.key_del[i]= HA_OFFSET_ERROR; + + info->state->key_file_length=share->base.keystart; + + lock_memory(param); /* Everything is alloced */ + while (!(error=sort_get_next_record(sort_info))) + { + if (writekeys(info,(byte*) sort_info->record,sort_info->filepos)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY) goto err; + DBUG_DUMP("record",(byte*) sort_info->record,share->base.pack_reclength); + mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s", + info->errkey+1, + llstr(sort_info->start_recpos,llbuff), + llstr(info->lastpos,llbuff2)); + if (param->testflag & T_VERBOSE) + { + VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey, + sort_info->record,0L)); + _mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey, + USE_WHOLE_KEY); + } + sort_info->dupp++; + if (rep_quick == 1) + { + param->error_printed=1; + goto err; + } + continue; + } + if (sort_write_record(sort_info)) + goto err; + } + if (error > 0 || write_data_suffix(param,info) || + flush_io_cache(&info->rec_cache) || param->read_cache.error < 0) + goto err; + + if (param->testflag & T_WRITE_LOOP) + { + VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); + } + if (my_chsize(share->kfile,info->state->key_file_length,MYF(0))) + { + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", + my_errno); + goto err; + } + + if (rep_quick && del+sort_info->dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without -q"); + got_error=1; + goto err; + } + + if (!rep_quick) + { + my_close(info->dfile,MYF(0)); + info->dfile=new_file; + info->state->data_file_length=sort_info->filepos; + /* Only whole records */ + share->state.split=info->state->records+info->state->del; + param->out_flag|=O_NEW_DATA; /* Data in new file */ + share->state.version=(ulong) time((time_t*) 0); /* Force reopen */ + } + else + info->state->data_file_length=sort_info->max_pos; + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info->dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info->dupp,llbuff)); + } + + got_error=0; + /* If invoked by external program that uses thr_lock */ + if (&share->state.state != info->state) + memcpy( &share->state.state, info->state, sizeof(*info->state)); + +err: + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d for record at pos %s",my_errno, + llstr(sort_info->start_recpos,llbuff)); + if (new_file >= 0) + { + VOID(my_close(new_file,MYF(0))); + VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks, + MYF(MY_WME))); + } + } + if (sort_info->record) + my_free(sort_info->record,MYF(0)); + + my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); + VOID(end_io_cache(¶m->read_cache)); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + VOID(end_io_cache(&info->rec_cache)); + got_error|=flush_blocks(param,share->kfile); + if (!got_error && param->testflag & T_UNPACK) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + share->data_file_type=sort_info->new_data_file_type; + } + DBUG_RETURN(got_error); +} /* rep */ + + +/* Uppate keyfile when doing repair */ + +static int writekeys(register MI_INFO *info,byte *buff,my_off_t filepos) +{ + register uint i; + uchar *key; + DBUG_ENTER("writekeys"); + + key=info->lastkey+info->s->base.max_key_length; + for (i=0 ; i < info->s->base.keys ; i++) + { + if (((ulonglong) 1 << i) & info->s->state.key_map) + { + uint key_length=_mi_make_key(info,i,key,buff,filepos); + if (_mi_ck_write(info,i,key,key_length)) goto err; + } + } + DBUG_RETURN(0); + + err: + if (my_errno == HA_ERR_FOUND_DUPP_KEY) + { + info->errkey=(int) i; /* This key was found */ + while ( i-- > 0 ) + { + if (((ulonglong) 1 << i) & info->s->state.key_map) + { + uint key_length=_mi_make_key(info,i,key,buff,filepos); + if (_mi_ck_delete(info,i,key,key_length)) break; + } + } + } + DBUG_PRINT("error",("errno: %d",my_errno)); + DBUG_RETURN(-1); +} /* writekeys */ + + + /* Change all key-pointers that points to a records */ + +int movepoint(register MI_INFO *info, byte *record, my_off_t oldpos, + my_off_t newpos, uint prot_key) +{ + register uint i; + uchar *key; + uint key_length; + DBUG_ENTER("movepoint"); + + key=info->lastkey+info->s->base.max_key_length; + for (i=0 ; i < info->s->base.keys; i++) + { + if (i != prot_key && (((ulonglong) 1 << i) & info->s->state.key_map)) + { + key_length=_mi_make_key(info,i,key,record,oldpos); + if (info->s->keyinfo[i].flag & HA_NOSAME) + { /* Change pointer direct */ + uint nod_flag; + MI_KEYDEF *keyinfo; + keyinfo=info->s->keyinfo+i; + if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY, + (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF), + info->s->state.key_root[i])) + DBUG_RETURN(-1); + nod_flag=mi_test_if_nod(info->buff); + _mi_dpointer(info,info->int_keypos-nod_flag- + info->s->rec_reflength,newpos); + if (_mi_write_keypage(info,keyinfo,info->last_keypage,info->buff)) + DBUG_RETURN(-1); + } + else + { /* Change old key to new */ + if (_mi_ck_delete(info,i,key,key_length)) + DBUG_RETURN(-1); + key_length=_mi_make_key(info,i,key,record,newpos); + if (_mi_ck_write(info,i,key,key_length)) + DBUG_RETURN(-1); + } + } + } + DBUG_RETURN(0); +} /* movepoint */ + + + /* Tell system that we want all memory for our cache */ + +void lock_memory(MI_CHECK *param __attribute__((unused))) +{ +#ifdef SUN_OS /* Key-cacheing thrases on sun 4.1 */ + if (param->opt_lock_memory) + { + int success = mlockall(MCL_CURRENT); /* or plock(DATLOCK); */ + if (geteuid() == 0 && success != 0) + mi_check_print_warning(param, + "Failed to lock memory. errno %d",my_errno); + } +#endif +} /* lock_memory */ + + + /* Flush all changed blocks to disk */ + +int flush_blocks(MI_CHECK *param, File file) +{ + if (flush_key_blocks(file,FLUSH_RELEASE)) + { + mi_check_print_error(param,"%d when trying to write bufferts",my_errno); + return(1); + } + if (!param->using_global_keycache) + end_key_cache(); + return 0; +} /* flush_blocks */ + + + /* Sort index for more efficent reads */ + +int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name) +{ + reg2 uint key; + reg1 MI_KEYDEF *keyinfo; + File new_file; + my_off_t index_pos[MI_MAX_POSSIBLE_KEY]; + DBUG_ENTER("sort_index"); + + if (!(param->testflag & T_SILENT)) + printf("- Sorting index for MyISAM-table '%s'\n",name); + + if ((new_file=my_create(fn_format(param->temp_filename,name,"", + INDEX_TMP_EXT,2+4), + 0,param->tmpfile_createflag,MYF(0))) <= 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + DBUG_RETURN(-1); + } + if (filecopy(param, new_file,info->s->kfile,0L, + (ulong) info->s->base.keystart, "headerblock")) + goto err; + + param->new_file_pos=info->s->base.keystart; + for (key= 0,keyinfo= &info->s->keyinfo[0]; key < info->s->base.keys ; + key++,keyinfo++) + { + if (!(((ulonglong) 1 << key) & info->s->state.key_map)) + continue; + + if (info->s->state.key_root[key] != HA_OFFSET_ERROR) + { + index_pos[key]=param->new_file_pos; /* Write first block here */ + if (sort_one_index(param,info,keyinfo,info->s->state.key_root[key], + new_file)) + goto err; + } + else + index_pos[key]= HA_OFFSET_ERROR; /* No blocks */ + } + + /* Flush key cache for this file if we are calling this outside myisamchk */ + flush_key_blocks(info->s->kfile, FLUSH_IGNORE_CHANGED); + + /* Put same locks as old file */ + if (lock_file(param,new_file,0L,F_WRLCK,"tempfile",param->temp_filename)) + goto err; + info->s->state.version=(ulong) time((time_t*) 0); + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + VOID(my_close(info->s->kfile,MYF(MY_WME))); + param->out_flag|=O_NEW_INDEX; /* Data in new file */ + + info->s->kfile=new_file; + info->state->key_file_length=param->new_file_pos; + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + for (key=0 ; key < info->s->base.keys ; key++) + info->s->state.key_root[key]=index_pos[key]; + for (key=0 ; key < info->s->state.header.max_block_size ; key++) + info->s->state.key_del[key]= HA_OFFSET_ERROR; + + DBUG_RETURN(0); + +err: + VOID(my_close(new_file,MYF(MY_WME))); + VOID(my_delete(param->temp_filename,MYF(MY_WME))); + DBUG_RETURN(-1); +} /* sort_index */ + + + /* Sort records recursive using one index */ + +static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t pagepos, File new_file) +{ + uint length,nod_flag,used_length; + uchar *buff,*keypos,*endpos; + uchar key[MI_MAX_POSSIBLE_KEY_BUFF]; + my_off_t new_page_pos,next_page; + char llbuff[22]; + DBUG_ENTER("sort_one_index"); + + new_page_pos=param->new_file_pos; + param->new_file_pos+=keyinfo->block_length; + + if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not Enough memory"); + DBUG_RETURN(-1); + } + if (!_mi_fetch_keypage(info,keyinfo,pagepos,buff,0)) + { + mi_check_print_error(param,"Can't read key block from filepos: %s", + llstr(pagepos,llbuff)); + goto err; + } + if ((nod_flag=mi_test_if_nod(buff))) + { + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + for ( ;; ) + { + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */ + if (sort_one_index(param,info,keyinfo,next_page, new_file)) + { + DBUG_PRINT("error",("From page: %ld, keyoffset: %d used_length: %d", + (ulong) pagepos, (int) (keypos - buff), + (int) used_length)); + DBUG_DUMP("buff",(byte*) buff,used_length); + goto err; + } + } + if (keypos >= endpos || + ((*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0) + break; +#ifdef EXTRA_DEBUG + assert(keypos <= endpos); +#endif + } + } + + /* Fill block with zero and write it to the new index file */ + length=mi_getint(buff); + bzero((byte*) buff+length,keyinfo->block_length-length); + if (my_pwrite(new_file,(byte*) buff,(uint) keyinfo->block_length, + new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL))) + { + mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno); + goto err; + } + my_afree((gptr) buff); + DBUG_RETURN(0); +err: + my_afree((gptr) buff); + DBUG_RETURN(1); +} /* sort_one_index */ + + + /* Change to use new file */ + /* Copy stats from old file to new file, deletes orginal and */ + /* changes new file name to old file name */ + +int change_to_newfile(const char * filename, const char * old_ext, + const char * new_ext, + uint raid_chunks __attribute__((unused))) +{ + char old_filename[FN_REFLEN],new_filename[FN_REFLEN]; +#ifdef USE_RAID + if (raid_chunks) + return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4), + fn_format(new_filename,filename,"",new_ext,2+4), + raid_chunks, + MYF(MY_WME+MY_LINK_WARNING)); +#endif + return my_redel(fn_format(old_filename,filename,"",old_ext,2+4), + fn_format(new_filename,filename,"",new_ext,2+4), + MYF(MY_WME+MY_LINK_WARNING)); +} /* change_to_newfile */ + + + /* Locks a whole file */ + /* Gives an error-message if file can't be locked */ + +int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type, + const char *filetype, const char *filename) +{ + if (my_lock(file,lock_type,start,F_TO_EOF, + param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) : + MYF(MY_SEEK_NOT_DONE | MY_DONT_WAIT))) + { + mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename); + param->error_printed=2; /* Don't give that data is crashed */ + return 1; + } + return 0; +} /* lock_file */ + + + /* Copy a block between two files */ + +int filecopy(MI_CHECK *param, File to,File from,my_off_t start, + my_off_t length, const char *type) +{ + char tmp_buff[IO_SIZE],*buff; + ulong buff_length; + DBUG_ENTER("filecopy"); + + buff_length=(ulong) min(param->write_buffer_length,length); + if (!(buff=my_malloc(buff_length,MYF(0)))) + { + buff=tmp_buff; buff_length=IO_SIZE; + } + + VOID(my_seek(from,start,MY_SEEK_SET,MYF(0))); + while (length > buff_length) + { + if (my_read(from,(byte*) buff,buff_length,MYF(MY_NABP)) || + my_write(to,(byte*) buff,buff_length,param->myf_rw)) + goto err; + length-= buff_length; + } + if (my_read(from,(byte*) buff,(uint) length,MYF(MY_NABP)) || + my_write(to,(byte*) buff,(uint) length,param->myf_rw)) + goto err; + if (buff != tmp_buff) + my_free(buff,MYF(0)); + DBUG_RETURN(0); +err: + if (buff != tmp_buff) + my_free(buff,MYF(0)); + mi_check_print_error(param,"Can't copy %s to tempfile, error %d", + type,my_errno); + DBUG_RETURN(1); +} + + /* Fix table using sorting */ + /* saves new table in temp_filename */ + +int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, + const char * name, int rep_quick) +{ + int got_error; + uint i; + ulong length; + ha_rows start_records; + my_off_t new_header_length,del; + File new_file; + MI_SORT_PARAM sort_param; + MYISAM_SHARE *share=info->s; + ulong *rec_per_key_part; + char llbuff[22]; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("rep_by_sort"); + + start_records=info->state->records; + got_error=1; + new_file= -1; + new_header_length=(param->testflag & T_UNPACK) ? 0 : + share->pack.header_length; + if (!(param->testflag & T_SILENT)) + { + printf("- recovering MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(start_records,llbuff)); + } + bzero((char*) sort_info,sizeof(*sort_info)); + if (!(sort_info->key_block= + alloc_key_blocks(param, + (uint) param->sort_key_blocks, + share->base.max_key_block_length)) + || init_io_cache(¶m->read_cache,info->dfile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) || + (! rep_quick && + init_io_cache(&info->rec_cache,info->dfile, + (uint) param->write_buffer_length, + WRITE_CACHE,new_header_length,1, + MYF(MY_WME | MY_WAIT_IF_FULL)))) + goto err; + sort_info->key_block_end=sort_info->key_block+param->sort_key_blocks; + info->opt_flag|=WRITE_CACHE_USED; + info->rec_cache.file=info->dfile; /* for sort_delete_record */ + + /* Flush key cache for this file if we are calling this outside myisamchk */ + flush_key_blocks(share->kfile, FLUSH_IGNORE_CHANGED); + + if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, + MYF(0)))) + { + mi_check_print_error(param,"Not enough memory for extra record"); + goto err; + } + if (!rep_quick) + { + if ((new_file=my_raid_create(fn_format(param->temp_filename,name,"", + DATA_TMP_EXT, + 2+4), + 0,param->tmpfile_createflag, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (filecopy(param, new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + share->state.dellink= HA_OFFSET_ERROR; + info->rec_cache.file=new_file; + } + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + for (i=0 ; i < share->base.keys ; i++) + share->state.key_root[i]= HA_OFFSET_ERROR; + for (i=0 ; i < share->state.header.max_block_size ; i++) + share->state.key_del[i]= HA_OFFSET_ERROR; + + info->state->key_file_length=share->base.keystart; + + sort_info->info=info; + sort_info->param = param; + + if ((sort_info->new_data_file_type=share->data_file_type) == + COMPRESSED_RECORD && param->testflag & T_UNPACK) + { + if (share->options & HA_OPTION_PACK_RECORD) + sort_info->new_data_file_type = DYNAMIC_RECORD; + else + sort_info->new_data_file_type = STATIC_RECORD; + } + + sort_info->filepos=new_header_length; + sort_info->dupp=0; + sort_info->buff=0; + param->read_cache.end_of_file=sort_info->filelength= + my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0)); + + if (share->data_file_type == DYNAMIC_RECORD) + length=max(share->base.min_pack_length+1,share->base.min_block_length); + else if (share->data_file_type == COMPRESSED_RECORD) + length=share->base.min_block_length; + else + length=share->base.pack_reclength; + sort_param.max_records=sort_info->max_records= + (ha_rows) (sort_info->filelength/length+1); + sort_param.key_cmp=sort_key_cmp; + sort_param.key_write=sort_key_write; + sort_param.key_read=sort_key_read; + sort_param.lock_in_memory=lock_memory; + sort_param.tmpdir=param->tmpdir; + sort_param.sort_info=sort_info; + + del=info->state->del; + if (sort_info->new_data_file_type != COMPRESSED_RECORD && + ! rep_quick) + share->state.checksum=0; + + rec_per_key_part= param->rec_per_key_part; + for (sort_info->key=0 ; sort_info->key < share->base.keys ; + rec_per_key_part+=sort_info->keyinfo->keysegs, sort_info->key++) + { + sort_info->keyinfo=share->keyinfo+sort_info->key; + if (!(((ulonglong) 1 << sort_info->key) & share->state.key_map)) + continue; + + if ((!(param->testflag & T_SILENT))) + printf ("- Fixing index %d\n",sort_info->key+1); + sort_info->max_pos=sort_info->pos=share->pack.header_length; + sort_info->keyseg=sort_info->keyinfo->seg; + sort_info->fix_datafile= (my_bool) (sort_info->key == 0 && ! rep_quick); + bzero((char*) sort_info->unique,sizeof(sort_info->unique)); + sort_param.key_length=share->rec_reflength; + for (i=0 ; sort_info->keyseg[i].type != HA_KEYTYPE_END; i++) + { + sort_param.key_length+=sort_info->keyseg[i].length; + if (sort_info->keyseg[i].flag & HA_SPACE_PACK) + sort_param.key_length+=get_pack_length(sort_info->keyseg[i].length); + if (sort_info->keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH)) + sort_param.key_length+=2 + test(sort_info->keyseg[i].length >= 127); + if (sort_info->keyseg[i].flag & HA_NULL_PART) + sort_param.key_length++; + } + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + + if (_create_index_by_sort(&sort_param, + (my_bool) (!(param->testflag & T_VERBOSE)), + (uint) param->sort_buffer_length)) + goto err; + + /* Set for next loop */ + sort_param.max_records=sort_info->max_records= + (ha_rows) info->state->records; + + if (param->testflag & T_STATISTICS) + update_key_parts(sort_info->keyinfo, rec_per_key_part, sort_info->unique, + (ulonglong) info->state->records); + + if (sort_info->fix_datafile) + { + param->read_cache.end_of_file=sort_info->filepos; + if (write_data_suffix(param,info) || end_io_cache(&info->rec_cache)) + goto err; + share->state.state.data_file_length = info->state->data_file_length + = sort_info->filepos; + /* Only whole records */ + share->state.split=info->state->records+info->state->del; + share->state.version=(ulong) time((time_t*) 0); + param->out_flag|=O_NEW_DATA; /* Data in new file */ + my_close(info->dfile,MYF(0)); + info->dfile=new_file; + share->data_file_type=sort_info->new_data_file_type; + share->pack.header_length=(ulong) new_header_length; + } + else + info->state->data_file_length=sort_info->max_pos; + + if (flush_pending_blocks(param)) + goto err; + + param->read_cache.file=info->dfile; /* re-init read cache */ + reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length,1, + 1); + } + + if (param->testflag & T_WRITE_LOOP) + { + VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); + } + + if (rep_quick && del+sort_info->dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without -q"); + got_error=1; + goto err; + } + + if (rep_quick != 1) + { + my_off_t skr=info->state->data_file_length+ + (share->options & HA_OPTION_COMPRESS_RECORD ? + MEMMAP_EXTRA_MARGIN : 0); +#ifdef USE_RELOC + if (share->data_file_type == STATIC_RECORD && + skr < share->base.reloc*share->base.min_pack_length) + skr=share->base.reloc*share->base.min_pack_length; +#endif + if (skr != sort_info->filelength && !info->s->base.raid_type) + if (my_chsize(info->dfile,skr,MYF(0))) + mi_check_print_warning(param, + "Can't change size of datafile, error: %d", + my_errno); + } + if (my_chsize(share->kfile,info->state->key_file_length,MYF(0))) + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", + my_errno); + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info->dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info->dupp,llbuff)); + } + got_error=0; + + if (&share->state.state != info->state) + memcpy( &share->state.state, info->state, sizeof(*info->state)); + +err: + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d when fixing table",my_errno); + if (new_file >= 0) + { + VOID(end_io_cache(&info->rec_cache)); + VOID(my_close(new_file,MYF(0))); + VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks, + MYF(MY_WME))); + } + } + my_free((gptr) sort_info->key_block,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_info->record,MYF(MY_ALLOW_ZERO_PTR)); + my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); + VOID(end_io_cache(¶m->read_cache)); + VOID(end_io_cache(&info->rec_cache)); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + if (!got_error && param->testflag & T_UNPACK) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + } + DBUG_RETURN(got_error); +} /* rep_by_sort */ + + + /* Read next record and return next key */ + +static int sort_key_read(SORT_INFO *sort_info, void *key) +{ + int error; + MI_INFO *info; + DBUG_ENTER("sort_key_read"); + + info=sort_info->info; + + if ((error=sort_get_next_record(sort_info))) + DBUG_RETURN(error); + if (info->state->records == sort_info->max_records) + { + mi_check_print_error(sort_info->param, + "Found too many records; Can`t continue"); + DBUG_RETURN(1); + } + VOID(_mi_make_key(info,sort_info->key,key,sort_info->record, + sort_info->filepos)); + DBUG_RETURN(sort_write_record(sort_info)); +} /* sort_key_read */ + + + /* Read next record from file using parameters in sort_info */ + /* Return -1 if end of file, 0 if ok and > 0 if error */ + +static int sort_get_next_record(SORT_INFO *sort_info) +{ + int searching; + uint found_record,b_type,left_length; + my_off_t pos; + byte *to; + MI_BLOCK_INFO block_info; + MI_INFO *info; + MYISAM_SHARE *share; + MI_CHECK *param=sort_info->param; + char llbuff[22],llbuff2[22]; + DBUG_ENTER("sort_get_next_record"); + + info=sort_info->info; + share=info->s; + switch (share->data_file_type) { + case STATIC_RECORD: + for (;;) + { + if (my_b_read(¶m->read_cache,sort_info->record, + share->base.pack_reclength)) + DBUG_RETURN(-1); + sort_info->start_recpos=sort_info->pos; + if (!sort_info->fix_datafile) + sort_info->filepos=sort_info->pos; + sort_info->max_pos=(sort_info->pos+=share->base.pack_reclength); + share->state.split++; + if (*sort_info->record) + DBUG_RETURN(0); + if (!sort_info->fix_datafile) + { + info->state->del++; + info->state->empty+=share->base.pack_reclength; + } + } + case DYNAMIC_RECORD: + LINT_INIT(to); + pos=sort_info->pos; + searching=(sort_info->fix_datafile && (param->testflag & T_EXTEND)); + for (;;) + { + found_record=block_info.second_read= 0; + left_length=1; + if (searching) + pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE); + do + { + if (pos > sort_info->max_pos) + sort_info->max_pos=pos; + if (pos & (MI_DYN_ALIGN_SIZE-1)) + { + if ((param->testflag & T_VERBOSE) || searching == 0) + { + mi_check_print_info(param,"Wrong aligned block at %s", + llstr(pos,llbuff)); + goto try_next; + } + } + if (found_record && pos == param->search_after_block) + mi_check_print_info(param,"Block: %s used by record at %s", + llstr(param->search_after_block,llbuff), + llstr(sort_info->start_recpos,llbuff2)); + if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header,pos, + MI_BLOCK_INFO_HEADER_LENGTH, + test(! found_record) | 2)) + { + if (found_record) + { + mi_check_print_info(param, + "Can't read whole record at %s (errno: %d)", + llstr(sort_info->start_recpos,llbuff),errno); + goto try_next; + } + DBUG_RETURN(-1); + } + if (searching && ! sort_info->fix_datafile) + { + param->error_printed=1; + DBUG_RETURN(1); /* Something wrong with data */ + } + if (((b_type=_mi_get_block_info(&block_info,-1,pos)) & + (BLOCK_ERROR | BLOCK_FATAL_ERROR)) || + ((b_type & BLOCK_FIRST) && + (block_info.rec_len < (uint) share->base.min_pack_length || + block_info.rec_len > (uint) share->base.max_pack_length))) + { + uint i; + if (param->testflag & T_VERBOSE || searching == 0) + mi_check_print_info(param, + "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped", + block_info.header[0],block_info.header[1], + block_info.header[2],llstr(pos,llbuff)); + if (found_record) + goto try_next; + block_info.second_read=0; + searching=1; + /* Search after block in read header string */ + for (i=MI_DYN_ALIGN_SIZE ; + i < MI_BLOCK_INFO_HEADER_LENGTH ; + i+= MI_DYN_ALIGN_SIZE) + if (block_info.header[i] >= 1 && + block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE) + break; + pos+=(ulong) i; + continue; + } + if (b_type & BLOCK_DELETED) + { + bool error=0; + if (block_info.block_len+ (uint) (block_info.filepos-pos) < + share->base.min_block_length) + { + if (!searching) + mi_check_print_info(param, + "Deleted block with impossible length %u at %s", + block_info.block_len,llstr(pos,llbuff)); + error=1; + } + else + { + if ((block_info.next_filepos != HA_OFFSET_ERROR && + block_info.next_filepos >= + info->state->data_file_length) || + (block_info.prev_filepos != HA_OFFSET_ERROR && + block_info.prev_filepos >= info->state->data_file_length)) + { + if (!searching) + mi_check_print_info(param, + "Delete link points outside datafile at %s", + llstr(pos,llbuff)); + error=1; + } + } + if (error) + { + if (found_record) + goto try_next; + searching=1; + pos++; + block_info.second_read=0; + continue; + } + } + else + { + if (block_info.block_len+ (uint) (block_info.filepos-pos) < + share->base.min_block_length || + block_info.block_len > (uint) share->base.max_pack_length+ + MI_SPLIT_LENGTH) + { + if (!searching) + mi_check_print_info(param, + "Found block with impossible length %u at %s; Skipped", + block_info.block_len+ (uint) (block_info.filepos-pos), + llstr(pos,llbuff)); + if (found_record) + goto try_next; + searching=1; + pos++; + block_info.second_read=0; + continue; + } + } + if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) + { + if (!sort_info->fix_datafile && (b_type & BLOCK_DELETED)) + { + info->state->empty+=block_info.block_len; + info->state->del++; + share->state.split++; + } + if (found_record) + goto try_next; + if (searching) + pos++; + else + pos=block_info.filepos+block_info.block_len; + block_info.second_read=0; + continue; + } + + share->state.split++; + if (! found_record++) + { + sort_info->find_length=left_length=block_info.rec_len; + sort_info->start_recpos=pos; + if (!sort_info->fix_datafile) + sort_info->filepos=sort_info->start_recpos; + if (sort_info->fix_datafile && (param->testflag & T_EXTEND)) + sort_info->pos=block_info.filepos+1; + else + sort_info->pos=block_info.filepos+block_info.block_len; + if (share->base.blobs) + { + if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + { + mi_check_print_error(param,"Not enough memory for blob at %s", + llstr(sort_info->start_recpos,llbuff)); + DBUG_RETURN(-1); + } + } + else + to= info->rec_buff; + } + if (left_length < block_info.data_len || ! block_info.data_len) + { + mi_check_print_info(param,"Found block with too small length at %s; Skipped", + llstr(sort_info->start_recpos,llbuff)); + goto try_next; + } + if (block_info.filepos + block_info.data_len > + param->read_cache.end_of_file) + { + mi_check_print_info(param,"Found block that points outside data file at %s", + llstr(sort_info->start_recpos,llbuff)); + goto try_next; + } + if (_mi_read_cache(¶m->read_cache,to,block_info.filepos, + block_info.data_len, test(found_record == 1))) + { + mi_check_print_info(param,"Read error for block at: %s (error: %d); Skipped", + llstr(block_info.filepos,llbuff),my_errno); + goto try_next; + } + left_length-=block_info.data_len; + to+=block_info.data_len; + pos=block_info.next_filepos; + if (pos == HA_OFFSET_ERROR && left_length) + { + mi_check_print_info(param,"Wrong block with wrong total length starting at %s", + llstr(sort_info->start_recpos,llbuff)); + goto try_next; + } + if (pos + MI_BLOCK_INFO_HEADER_LENGTH > param->read_cache.end_of_file) + { + mi_check_print_info(param,"Found link that points at %s (outside data file) at %s", + llstr(pos,llbuff2), llstr(sort_info->start_recpos,llbuff)); + goto try_next; + } + } while (left_length); + + if (_mi_rec_unpack(info,sort_info->record,info->rec_buff, + sort_info->find_length) != MY_FILE_ERROR) + { + if (param->read_cache.error < 0) + DBUG_RETURN(1); + if ((param->testflag & (T_EXTEND | T_REP)) || searching) + { + if (info->s->calc_checksum) + info->checksum=mi_checksum(info,sort_info->record); + if (_mi_rec_check(info, sort_info->record)) + { + mi_check_print_info(param,"Found wrong packed record at %s", + llstr(sort_info->start_recpos,llbuff)); + goto try_next; + } + } + DBUG_RETURN(0); + } + try_next: + pos=(sort_info->start_recpos+=MI_DYN_ALIGN_SIZE); + searching=1; + } + case COMPRESSED_RECORD: + for (searching=0 ;; searching=1, sort_info->pos++) + { + if (_mi_read_cache(¶m->read_cache,(byte*) block_info.header,sort_info->pos, + share->pack.ref_length,1)) + DBUG_RETURN(-1); + if (searching && ! sort_info->fix_datafile) + { + param->error_printed=1; + DBUG_RETURN(1); /* Something wrong with data */ + } + sort_info->start_recpos=sort_info->pos; + if (_mi_pack_get_block_info(info,&block_info,-1,sort_info->pos, NullS)) + DBUG_RETURN(-1); + if (!block_info.rec_len && + sort_info->pos + MEMMAP_EXTRA_MARGIN == + param->read_cache.end_of_file) + DBUG_RETURN(-1); + if (block_info.rec_len < (uint) share->min_pack_length || + block_info.rec_len > (uint) share->max_pack_length) + { + if (! searching) + mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n", + block_info.rec_len, llstr(sort_info->pos,llbuff)); + continue; + } + if (_mi_read_cache(¶m->read_cache,(byte*) info->rec_buff, + block_info.filepos, block_info.rec_len,1)) + { + if (! searching) + mi_check_print_info(param,"Couldn't read hole record from %s", + llstr(sort_info->pos,llbuff)); + continue; + } + if (_mi_pack_rec_unpack(info,sort_info->record,info->rec_buff, + block_info.rec_len)) + { + if (! searching) + mi_check_print_info(param,"Found wrong record at %s", llstr(sort_info->pos,llbuff)); + continue; + } + if (!sort_info->fix_datafile) + sort_info->filepos=sort_info->pos; + sort_info->max_pos=(sort_info->pos=block_info.filepos+ + block_info.rec_len); + share->state.split++; + info->packed_length=block_info.rec_len; + DBUG_RETURN(0); + } + } + DBUG_RETURN(1); /* Impossible */ +} + + + /* Write record to new file */ + +int sort_write_record(SORT_INFO *sort_info) +{ + int flag; + uint length; + ulong block_length,reclength; + byte *from; + byte block_buff[8]; + MI_INFO *info; + MYISAM_SHARE *share; + MI_CHECK *param=sort_info->param; + DBUG_ENTER("sort_write_record"); + + info=sort_info->info; + share=info->s; + if (sort_info->fix_datafile) + { + switch (sort_info->new_data_file_type) { + case STATIC_RECORD: + if (my_b_write(&info->rec_cache,sort_info->record, + share->base.pack_reclength)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_info->filepos+=share->base.pack_reclength; + info->s->state.checksum+=mi_static_checksum(info, sort_info->record); + break; + case DYNAMIC_RECORD: + if (! info->blobs) + from=info->rec_buff; + else + { + /* must be sure that local buffer is big enough */ + reclength=info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,sort_info->record)+ + ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER; + if (sort_info->buff_length < reclength) + { + if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength, + MYF(MY_FREE_ON_ERROR | + MY_ALLOW_ZERO_PTR)))) + DBUG_RETURN(1); + sort_info->buff_length=reclength; + } + from=sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER); + } + info->checksum=mi_checksum(info,sort_info->record); + reclength=_mi_rec_pack(info,from,sort_info->record); + info->s->state.checksum+=info->checksum; + block_length=reclength+ 3 +test(reclength > 65532L); + if (block_length < share->base.min_block_length) + block_length=share->base.min_block_length; + flag=0; + info->update|=HA_STATE_WRITE_AT_END; + block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE); + if (_mi_write_part_record(info,0L,block_length,HA_OFFSET_ERROR, + &from,&reclength,&flag)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_info->filepos+=block_length; + break; + case COMPRESSED_RECORD: + reclength=info->packed_length; + length=save_pack_length(block_buff,reclength); + if (info->s->base.blobs) + length+=save_pack_length(block_buff+length,info->blob_length); + if (my_b_write(&info->rec_cache,block_buff,length) || + my_b_write(&info->rec_cache,(byte*) info->rec_buff,reclength)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_info->filepos+=reclength+length; + break; + } + } + info->state->records++; + if ((param->testflag & T_WRITE_LOOP) && + (info->state->records % WRITE_COUNT) == 0) + { + char llbuff[22]; + printf("%s\r", llstr(info->state->records,llbuff)); VOID(fflush(stdout)); + } + DBUG_RETURN(0); +} /* sort_write_record */ + + + /* Compare two keys from _create_index_by_sort */ + +static int sort_key_cmp(SORT_INFO *sort_info, const void *a, const void *b) +{ + uint not_used; + return (_mi_key_cmp(sort_info->keyseg,*((uchar**) a),*((uchar**) b), + USE_WHOLE_KEY, SEARCH_SAME,¬_used)); +} /* sort_key_cmp */ + + +static int sort_key_write(SORT_INFO *sort_info, const void *a) +{ + uint diff_pos; + char llbuff[22],llbuff2[22]; + MI_CHECK *param= sort_info->param; + int cmp; + + if (sort_info->key_block->inited) + { + cmp=_mi_key_cmp(sort_info->keyseg,sort_info->key_block->lastkey,(uchar*) a, + USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE ,&diff_pos); + } + else + { + cmp= -1; diff_pos=sort_info->keyinfo->keysegs; + } + if ((sort_info->keyinfo->flag & HA_NOSAME) && cmp == 0) + { + sort_info->dupp++; + sort_info->info->lastpos=get_record_for_key(sort_info->info, + sort_info->keyinfo, + (uchar*) a); + mi_check_print_warning(param, + "Duplicate key for record at %10s against record at %10s", + llstr(sort_info->info->lastpos,llbuff), + llstr(get_record_for_key(sort_info->info, + sort_info->keyinfo, + sort_info->key_block-> + lastkey), + llbuff2)); + if (sort_info->param->testflag & T_VERBOSE) + _mi_print_key(stdout,sort_info->keyseg,(uchar*) a, USE_WHOLE_KEY); + return (sort_delete_record(param)); + } + sort_info->unique[diff_pos-1]++; +#ifndef DBUG_OFF + if (cmp > 0) + { + mi_check_print_error(param, + "Internal error: Keys are not in order from sort"); + return(1); + } +#endif + return (sort_insert_key(param,sort_info->key_block,(uchar*) a, + HA_OFFSET_ERROR)); +} /* sort_key_write */ + + + /* get pointer to record from a key */ + +static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key) +{ + return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key)); +} /* get_record_for_key */ + + + /* Insert a key in sort-key-blocks */ + +static int sort_insert_key(MI_CHECK *param, + register SORT_KEY_BLOCKS *key_block, uchar *key, + my_off_t prev_block) +{ + uint a_length,t_length,nod_flag; + my_off_t filepos; + uchar *anc_buff,*lastkey; + MI_KEY_PARAM s_temp; + MI_INFO *info; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("sort_insert_key"); + + anc_buff=key_block->buff; + info=sort_info->info; + lastkey=key_block->lastkey; + nod_flag= (key_block == sort_info->key_block ? 0 : + sort_info->info->s->base.key_reflength); + + if (!key_block->inited) + { + key_block->inited=1; + if (key_block == sort_info->key_block_end) + { + mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks"); + DBUG_RETURN(1); + } + a_length=2+nod_flag; + key_block->end_pos=anc_buff+2; + lastkey=0; /* No previous key in block */ + } + else + a_length=mi_getint(anc_buff); + + /* Save pointer to previous block */ + if (nod_flag) + _mi_kpointer(info,key_block->end_pos,prev_block); + + t_length=(*sort_info->keyinfo->pack_key)(sort_info->keyinfo,nod_flag, + (uchar*) 0,lastkey,lastkey,key, + &s_temp); + (*sort_info->keyinfo->store_key)(sort_info->keyinfo, + key_block->end_pos+nod_flag,&s_temp); + a_length+=t_length; + mi_putint(anc_buff,a_length,nod_flag); + key_block->end_pos+=t_length; + if (a_length <= sort_info->keyinfo->block_length) + { + VOID(_mi_move_key(sort_info->keyinfo,key_block->lastkey,key)); + key_block->last_length=a_length-t_length; + DBUG_RETURN(0); + } + + /* Fill block with end-zero and write filled block */ + mi_putint(anc_buff,key_block->last_length,nod_flag); + bzero((byte*) anc_buff+key_block->last_length, + sort_info->keyinfo->block_length- key_block->last_length); + if ((filepos=_mi_new(info,sort_info->keyinfo)) == HA_OFFSET_ERROR) + return 1; + if (my_pwrite(info->s->kfile,(byte*) anc_buff, + (uint) sort_info->keyinfo->block_length,filepos, + param->myf_rw)) + DBUG_RETURN(1); + DBUG_DUMP("buff",(byte*) anc_buff,mi_getint(anc_buff)); + + /* Write separator-key to block in next level */ + if (sort_insert_key(param,key_block+1,key_block->lastkey,filepos)) + DBUG_RETURN(1); + + /* clear old block and write new key in it */ + key_block->inited=0; + DBUG_RETURN(sort_insert_key(param, key_block,key,prev_block)); +} /* sort_insert_key */ + + + /* Delete record when we found a duplicated key */ + +static int sort_delete_record(MI_CHECK *param) +{ + uint i; + int old_file,error; + uchar *key; + MI_INFO *info; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("sort_delete_record"); + + if (param->opt_rep_quick == 1) + { + mi_check_print_error(param, + "Quick-recover aborted; Run recovery without switch 'q' or with switch -qq"); + DBUG_RETURN(1); + } + info=sort_info->info; + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + { + mi_check_print_error(param, + "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);; + DBUG_RETURN(1); + } + + old_file=info->dfile; + info->dfile=info->rec_cache.file; + if (sort_info->key) + { + key=info->lastkey+info->s->base.max_key_length; + if ((error=(*info->s->read_rnd)(info,sort_info->record,info->lastpos,0)) && + error != HA_ERR_RECORD_DELETED) + { + mi_check_print_error(param,"Can't read record to be removed"); + info->dfile=old_file; + DBUG_RETURN(1); + } + + for (i=0 ; i < sort_info->key ; i++) + { + uint key_length=_mi_make_key(info,i,key,sort_info->record,info->lastpos); + if (_mi_ck_delete(info,i,key,key_length)) + { + mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1); + info->dfile=old_file; + DBUG_RETURN(1); + } + } + if (info->s->calc_checksum) + info->s->state.checksum-=(*info->s->calc_checksum)(info, + sort_info->record); + } + error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info); + info->dfile=old_file; /* restore actual value */ + info->state->records--; + DBUG_RETURN(error); +} /* sort_delete_record */ + + + /* Fix all pending blocks and flush everything to disk */ + +static int flush_pending_blocks(MI_CHECK *param) +{ + uint nod_flag,length; + my_off_t filepos; + MI_INFO *info; + SORT_KEY_BLOCKS *key_block; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("flush_pending_blocks"); + + filepos= HA_OFFSET_ERROR; /* if empty file */ + info=sort_info->info; + nod_flag=0; + for (key_block=sort_info->key_block ; key_block->inited ; key_block++) + { + key_block->inited=0; + length=mi_getint(key_block->buff); + if (nod_flag) + _mi_kpointer(info,key_block->end_pos,filepos); + if ((filepos=_mi_new(info,sort_info->keyinfo)) == HA_OFFSET_ERROR) + DBUG_RETURN(1); + bzero((byte*) key_block->buff+length, + sort_info->keyinfo->block_length-length); + if (my_pwrite(info->s->kfile,(byte*) key_block->buff, + (uint) sort_info->keyinfo->block_length,filepos, + param->myf_rw)) + DBUG_RETURN(1); + DBUG_DUMP("buff",(byte*) key_block->buff,length); + nod_flag=1; + } + info->s->state.key_root[sort_info->key]=filepos; /* Last is root for tree */ + DBUG_RETURN(0); +} /* flush_pending_blocks */ + + + /* alloc space and pointers for key_blocks */ + +static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks, + uint buffer_length) +{ + reg1 uint i; + SORT_KEY_BLOCKS *block; + DBUG_ENTER("alloc_key_blocks"); + + if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+ + buffer_length+IO_SIZE)*blocks, + MYF(0)))) + { + mi_check_print_error(param,"Not Enough memory for sort-key-blocks"); + return(0); + } + for (i=0 ; i < blocks ; i++) + { + block[i].inited=0; + block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i; + } + DBUG_RETURN(block); +} /* alloc_key_blocks */ + + + /* Check if file is almost full */ + +int test_if_almost_full(MI_INFO *info) +{ + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + return 0; + return (my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0))/10*9 > + (my_off_t) (info->s->base.max_key_file_length) || + my_seek(info->dfile,0L,MY_SEEK_END,MYF(0))/10*9 > + (my_off_t) info->s->base.max_data_file_length); +} + + /* Recreate table with bigger more alloced record-data */ + +int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename) +{ + int error; + MI_INFO info; + MYISAM_SHARE share; + MI_KEYDEF *keyinfo,*key,*key_end; + MI_KEYSEG *keysegs,*keyseg; + MI_COLUMNDEF *recdef,*rec,*end; + MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end; + MI_STATUS_INFO status_info; + uint unpack,key_parts; + ha_rows max_records; + char name[FN_REFLEN]; + ulonglong file_length,tmp_length; + MI_CREATE_INFO create_info; + + error=1; /* Default error */ + info= **org_info; + status_info= (*org_info)->state[0]; + info.state= &status_info; + share= *(*org_info)->s; + unpack= (share.options & HA_OPTION_COMPRESS_RECORD) && + (param->testflag & T_UNPACK); + if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys))) + return 0; + memcpy((byte*) keyinfo,(byte*) share.keyinfo, + (size_t) (sizeof(MI_KEYDEF)*share.base.keys)); + + key_parts= share.base.all_key_parts; + if (!(keysegs=(MI_KEYSEG*) my_alloca(sizeof(MI_KEYSEG)* + (key_parts+share.base.keys)))) + { + my_afree((gptr) keyinfo); + return 1; + } + if (!(recdef=(MI_COLUMNDEF*) + my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1)))) + { + my_afree((gptr) keyinfo); + my_afree((gptr) keysegs); + return 1; + } + if (!(uniquedef=(MI_UNIQUEDEF*) + my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1)))) + { + my_afree((gptr) recdef); + my_afree((gptr) keyinfo); + my_afree((gptr) keysegs); + return 1; + } + + /* Copy the column definitions */ + memcpy((byte*) recdef,(byte*) share.rec, + (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1))); + for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++) + { + if (unpack && !(share.options & HA_OPTION_PACK_RECORD) && + rec->type != FIELD_BLOB && + rec->type != FIELD_VARCHAR && + rec->type != FIELD_CHECK) + rec->type=(int) FIELD_NORMAL; + } + + /* Change the new key to point at the saved key segments */ + memcpy((byte*) keysegs,(byte*) share.keyparts, + (size_t) (sizeof(MI_KEYSEG)*(key_parts+share.base.keys+ + share.state.header.uniques))); + keyseg=keysegs; + for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++) + { + key->seg=keyseg; + for (; keyseg->type ; keyseg++) + { + if (param->language) + keyseg->language=param->language; /* change language */ + } + keyseg++; /* Skipp end pointer */ + } + + /* Copy the unique definitions and change them to point at the new key + segments*/ + memcpy((byte*) uniquedef,(byte*) share.uniqueinfo, + (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques))); + for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques; + u_ptr != u_end ; u_ptr++) + { + u_ptr->seg=keyseg; + keyseg+=u_ptr->keysegs+1; + } + if (share.options & HA_OPTION_COMPRESS_RECORD) + share.base.records=max_records=info.state->records; + else if (share.base.min_pack_length) + max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) / + (ulong) share.base.min_pack_length); + else + max_records=0; + unpack= (share.options & HA_OPTION_COMPRESS_RECORD) && + (param->testflag & T_UNPACK); + share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD; + + file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)); + tmp_length= file_length+file_length/10; + set_if_bigger(file_length,param->max_data_file_length); + set_if_bigger(file_length,tmp_length); + set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length); + + VOID(mi_close(*org_info)); + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=max(max_records,share.base.records); + create_info.reloc_rows=share.base.reloc; + create_info.old_options=(share.options | + (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0)); + + create_info.data_file_length=file_length; + create_info.auto_increment=share.state.auto_increment; + create_info.raid_type= share.base.raid_type; + create_info.raid_chunks= share.base.raid_chunks; + create_info.raid_chunksize= share.base.raid_chunksize; + create_info.language = (param->language ? param->language : + share.state.header.language); + + if (mi_create(fn_format(name,filename,"",MI_NAME_IEXT, + 4+ (param->opt_follow_links ? 16 : 0)), + share.base.keys - share.state.header.uniques, + keyinfo, share.base.fields, recdef, + share.state.header.uniques, uniquedef, + &create_info, + HA_DONT_TOUCH_DATA)) + { + mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno); + goto end; + } + *org_info=mi_open(name,O_RDWR, + (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED : + (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED : + HA_OPEN_ABORT_IF_LOCKED); + if (!*org_info) + { + mi_check_print_error(param,"Got error %d when trying to open re-created indexfile", + my_errno); + goto end; + } + /* We are modifing */ + (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA; + VOID(_mi_readinfo(*org_info,F_WRLCK,0)); + (*org_info)->state->records=info.state->records; + if (share.state.create_time) + (*org_info)->s->state.create_time=share.state.create_time; + (*org_info)->s->state.unique=(*org_info)->this_unique= + share.state.unique; + (*org_info)->s->state.checksum=share.state.checksum; + (*org_info)->state->del=info.state->del; + (*org_info)->s->state.dellink=share.state.dellink; + (*org_info)->state->empty=info.state->empty; + (*org_info)->state->data_file_length=info.state->data_file_length; + if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT | + UPDATE_OPEN_COUNT)) + goto end; + error=0; +end: + my_afree((gptr) uniquedef); + my_afree((gptr) keyinfo); + my_afree((gptr) recdef); + my_afree((gptr) keysegs); + return error; +} + + + /* write suffix to data file if neaded */ + +int write_data_suffix(MI_CHECK *param, MI_INFO *info) +{ + if (info->s->options & HA_OPTION_COMPRESS_RECORD && + param->sort_info.fix_datafile) + { + char buff[MEMMAP_EXTRA_MARGIN]; + bzero(buff,sizeof(buff)); + if (my_b_write(&info->rec_cache,buff,sizeof(buff))) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + return 1; + } + param->read_cache.end_of_file+=sizeof(buff); + } + return 0; +} + + + /* Update state and myisamchk_time of indexfile */ + +int update_state_info(MI_CHECK *param, MI_INFO *info,uint update) +{ + MYISAM_SHARE *share=info->s; + + if (update & UPDATE_OPEN_COUNT) + { + share->state.open_count=0; + share->global_changed=0; + } + if (update & UPDATE_STAT) + { + uint key_parts= mi_uint2korr(share->state.header.key_parts); + share->state.rec_per_key_rows=info->state->records; + memcpy((char*) share->state.rec_per_key_part, + (char*) param->rec_per_key_part, + sizeof(*param->rec_per_key_part)*key_parts); + } + if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC)) + { + if (update & UPDATE_TIME) + { + share->state.check_time= (long) time((time_t*) 0); + if (!share->state.create_time) + share->state.create_time=share->state.check_time; + } + if (mi_state_info_write(share->kfile,&share->state,1+2)) + goto err; + } + { /* Force update of status */ + int error; + uint r_locks=share->r_locks,w_locks=share->w_locks; + share->r_locks=share->w_locks=0; + error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK); + share->r_locks=r_locks; share->w_locks=w_locks; + if (!error) + return 0; + } +err: + mi_check_print_error(param,"%d when updateing keyfile",my_errno); + return 1; +} + + /* + Update auto increment value for a table + When setting the 'repair_only' flag we only want to change the + old auto_increment value if its wrong (smaller than some given key). + The reason is that we shouldn't change the auto_increment value + for a table without good reason when only doing a repair; If the + user have inserted and deleted rows, the auto_increment value + may be bigger than the biggest current row and this is ok. + + If repair_only is not set, we will update the flag to the value in + param->auto_increment is bigger than the biggest key. + */ + +void update_auto_increment_key(MI_CHECK *param, MI_INFO *info, + my_bool repair_only) +{ + if (!info->s->base.auto_key || + !(((ulonglong) 1 << (info->s->base.auto_key-1) + & info->s->state.key_map))) + { + mi_check_print_info(param,"Table: %s doesn't have an auto increment key\n", + param->isam_file_name); + return; + } + if (!(param->testflag & T_SILENT) && + !(param->testflag & (T_REP | T_REP_BY_SORT))) + printf("Updating MyISAM file: %s\n", param->isam_file_name); + /* We have to use keyread here as a normal read uses info->rec_buff */ + mi_extra(info,HA_EXTRA_KEYREAD); + if (mi_rlast(info,info->rec_buff, info->s->base.auto_key-1)) + { + if (my_errno != HA_ERR_END_OF_FILE) + { + mi_extra(info,HA_EXTRA_NO_KEYREAD); + mi_check_print_error(param,"%d when reading last record",my_errno); + return; + } + if (!repair_only) + info->s->state.auto_increment=param->auto_increment_value; + } + else + { + ulonglong auto_increment= (repair_only ? info->s->state.auto_increment : + param->auto_increment_value); + info->s->state.auto_increment=0; + update_auto_increment(info,info->rec_buff); + set_if_bigger(info->s->state.auto_increment,auto_increment); + } + mi_extra(info,HA_EXTRA_NO_KEYREAD); + update_state_info(param, info, UPDATE_AUTO_INC); + return; +} + + /* calculate unique keys for each part key */ + +static void update_key_parts(MI_KEYDEF *keyinfo, + ulong *rec_per_key_part, + ulonglong *unique, + ulonglong records) +{ + ulonglong count=0,tmp; + uint parts; + for (parts=0 ; parts < keyinfo->keysegs ; parts++) + { + count+=unique[parts]; + if (count == 0) + tmp=records; + else + tmp= (records+count/2) / count; + if (tmp >= (ulonglong) ~(ulong) 0) + tmp=(ulonglong) ~(ulong) 0; + *rec_per_key_part=(ulong) tmp; + rec_per_key_part++; + } +} + + +ha_checksum mi_byte_checksum(const byte *buf, uint length) +{ + ha_checksum crc; + const byte *end=buf+length; + for (crc=0; buf != end; buf++) + crc=((crc << 1) + *((uchar*) buf)) + + test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1))); + return crc; +} diff --git a/myisam/mi_checksum.c b/myisam/mi_checksum.c new file mode 100644 index 00000000000..9dabe55e239 --- /dev/null +++ b/myisam/mi_checksum.c @@ -0,0 +1,67 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Calculate a checksum for a row */ + +#include "myisamdef.h" + +ha_checksum mi_checksum(MI_INFO *info, const byte *buf) +{ + uint i; + ha_checksum crc=0; + MI_COLUMNDEF *rec=info->s->rec; + + for (i=info->s->base.fields ; i-- ; buf+=(rec++)->length) + { + const byte *pos; + const byte *end; + switch (rec->type) { + case FIELD_BLOB: + { + ulong length=_mi_calc_blob_length(rec->length- + mi_portable_sizeof_char_ptr, + buf); + memcpy((char*) &pos, buf+rec->length- mi_portable_sizeof_char_ptr, + sizeof(char*)); + end=pos+length; + break; + } + case FIELD_VARCHAR: + { + uint length; + length=uint2korr(buf); + pos=buf+2; end=pos+length; + break; + } + default: + pos=buf; end=buf+rec->length; + break; + } + for ( ; pos != end ; pos++) + crc=((crc << 8) + *((uchar*) pos)) + (crc >> (8*sizeof(ha_checksum)-8)); + } + return crc; +} + + +ha_checksum mi_static_checksum(MI_INFO *info, const byte *pos) +{ + ha_checksum crc; + const byte *end=pos+info->s->base.reclength; + for (crc=0; pos != end; pos++) + crc=((crc << 8) + *((uchar*) pos)) + (crc >> (8*sizeof(ha_checksum)-8)); + return crc; +} diff --git a/myisam/mi_close.c b/myisam/mi_close.c new file mode 100644 index 00000000000..ebd64aa2349 --- /dev/null +++ b/myisam/mi_close.c @@ -0,0 +1,105 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* close a isam-database */ +/* + TODO: + We need to have a separate mutex on the closed file to allow other threads + to open other files during the time we flush the cache and close this file +*/ + +#include "myisamdef.h" + +int mi_close(register MI_INFO *info) +{ + int error=0,flag; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_close"); + DBUG_PRINT("enter",("base: %lx reopen: %u locks: %u", + info,(uint) share->reopen, + (uint) (share->w_locks+share->r_locks))); + + pthread_mutex_lock(&THR_LOCK_myisam); + if (info->lock_type == F_EXTRA_LCK) + info->lock_type=F_UNLCK; /* HA_EXTRA_NO_USER_CHANGE */ + + if (share->reopen == 1) + _mi_decrement_open_count(info); + + if (info->lock_type != F_UNLCK) + { + if (mi_lock_database(info,F_UNLCK)) + error=my_errno; + } + pthread_mutex_lock(&share->intern_lock); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + share->r_locks--; + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + if (end_io_cache(&info->rec_cache)) + error=my_errno; + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + } + flag= !--share->reopen; + myisam_open_list=list_delete(myisam_open_list,&info->open_list); + pthread_mutex_unlock(&share->intern_lock); + + if (flag) + { + if (share->kfile >= 0 && + flush_key_blocks(share->kfile, + share->temporary ? FLUSH_IGNORE_CHANGED : + FLUSH_RELEASE)) + error=my_errno; + if (share->kfile >= 0 && my_close(share->kfile,MYF(0))) + error = my_errno; +#ifdef HAVE_MMAP + if (share->file_map) + _mi_unmap_file(info); +#endif + if (share->decode_trees) + { + my_free((gptr) share->decode_trees,MYF(0)); + my_free((gptr) share->decode_tables,MYF(0)); + } +#ifdef THREAD + thr_lock_delete(&share->lock); + VOID(pthread_mutex_destroy(&share->intern_lock)); + { + int i,keys; + keys = share->state.header.keys; + for(i=0; i<keys; i++) { + VOID(rwlock_destroy(&share->key_root_lock[i])); + } + } +#endif + my_free((gptr) info->s,MYF(0)); + } + pthread_mutex_unlock(&THR_LOCK_myisam); + if (info->dfile >= 0 && my_close(info->dfile,MYF(0))) + error = my_errno; + + myisam_log_command(MI_LOG_CLOSE,info,NULL,0,error); + my_free((gptr) info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr) info,MYF(0)); + + if (error) + { + DBUG_RETURN(my_errno=error); + } + DBUG_RETURN(0); +} /* mi_close */ diff --git a/myisam/mi_create.c b/myisam/mi_create.c new file mode 100644 index 00000000000..27c47c5170b --- /dev/null +++ b/myisam/mi_create.c @@ -0,0 +1,628 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Create a MyISAM table */ + +#include "fulltext.h" +#if defined(MSDOS) || defined(__WIN__) +#ifdef __WIN__ +#include <fcntl.h> +#else +#include <process.h> /* Prototype for getpid */ +#endif +#endif +#include <m_ctype.h> + + /* + ** Old options is used when recreating database, from isamchk + */ + +int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, + uint columns, MI_COLUMNDEF *recinfo, + uint uniques, MI_UNIQUEDEF *uniquedefs, + MI_CREATE_INFO *ci,uint flags) +{ + register uint i,j; + File dfile,file; + int errpos,save_errno; + uint fields,length,max_key_length,packed,pointer, + key_length,info_length,key_segs,options,min_key_length_skipp, + base_pos,varchar_count,long_varchar_count,varchar_length, + max_key_block_length,unique_key_parts,offset; + ulong reclength, real_reclength,min_pack_length; + char buff[max(FN_REFLEN,2048)]; + ulong pack_reclength; + ulonglong tot_length,max_rows; + enum en_fieldtype type; + MYISAM_SHARE share; + MI_KEYDEF *keydef,tmp_keydef; + MI_UNIQUEDEF *uniquedef; + MI_KEYSEG *keyseg,tmp_keyseg; + MI_COLUMNDEF *rec; + ulong rec_per_key_part[MI_MAX_POSSIBLE_KEY*MI_MAX_KEY_SEG]; + my_off_t key_root[MI_MAX_POSSIBLE_KEY],key_del[MI_MAX_KEY_BLOCK_SIZE]; + MI_CREATE_INFO tmp_create_info; + DBUG_ENTER("mi_create"); + + if (!ci) + { + bzero((char*) &tmp_create_info,sizeof(tmp_create_info)); + ci=&tmp_create_info; + } + + if (keys + uniques > MI_MAX_KEY) + { + DBUG_RETURN(my_errno=HA_WRONG_CREATE_OPTION); + } + LINT_INIT(dfile); + LINT_INIT(file); + pthread_mutex_lock(&THR_LOCK_myisam); + errpos=0; + options=0; + bzero((byte*) &share,sizeof(share)); + + if (flags & HA_DONT_TOUCH_DATA) + { + if (!(ci->old_options & HA_OPTION_TEMP_COMPRESS_RECORD)) + options=ci->old_options & + (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD | + HA_OPTION_READ_ONLY_DATA | HA_OPTION_CHECKSUM | + HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE); + else + options=ci->old_options & + (HA_OPTION_CHECKSUM | HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE); + } + + if (ci->reloc_rows > ci->max_rows) + ci->reloc_rows=ci->max_rows; /* Check if wrong parameter */ + + /* Start by checking fields and field-types used */ + + reclength=varchar_count=varchar_length=long_varchar_count=packed= + min_pack_length=pack_reclength=0; + for (rec=recinfo, fields=0 ; + fields != columns ; + rec++,fields++) + { + reclength+=rec->length; + if ((type=(enum en_fieldtype) rec->type) != FIELD_NORMAL && + type != FIELD_CHECK) + { + packed++; + if (type == FIELD_BLOB) + { + share.base.blobs++; + if (pack_reclength != INT_MAX32) + { + if (rec->length == 4+mi_portable_sizeof_char_ptr) + pack_reclength= INT_MAX32; + else + pack_reclength+=(1 << ((rec->length-mi_portable_sizeof_char_ptr)*8)); /* Max blob length */ + } + } + else if (type == FIELD_SKIPP_PRESPACE || + type == FIELD_SKIPP_ENDSPACE) + { + if (pack_reclength != INT_MAX32) + pack_reclength+= rec->length > 255 ? 2 : 1; + min_pack_length++; + } + else if (type == FIELD_VARCHAR) + { + varchar_count++; + varchar_length+=rec->length-2; + packed--; + pack_reclength+=1; + if (test(rec->length > 257)) + { /* May be packed on 3 bytes */ + long_varchar_count++; + pack_reclength+=2; + } + } + else if (type != FIELD_SKIPP_ZERO) + { + min_pack_length+=rec->length; + packed--; /* Not a pack record type */ + } + } + else /* FIELD_NORMAL */ + min_pack_length+=rec->length; + } + if ((packed & 7) == 1) + { /* Bad packing, try to remove a zero-field */ + while (rec != recinfo) + { + rec--; + if (rec->type == (int) FIELD_SKIPP_ZERO && rec->length == 1) + { + rec->type=(int) FIELD_NORMAL; + packed--; + min_pack_length++; + break; + } + } + } + + if (packed || (flags & HA_PACK_RECORD)) + options|=HA_OPTION_PACK_RECORD; /* Must use packed records */ + if (options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) + min_pack_length+=varchar_count; /* Min length to pack */ + else + { + min_pack_length+=varchar_length+2*varchar_count; + } + if (flags & HA_CREATE_TMP_TABLE) + options|= HA_OPTION_TMP_TABLE; + if (flags & HA_CREATE_CHECKSUM || (options & HA_OPTION_CHECKSUM)) + { + options|= HA_OPTION_CHECKSUM; + min_pack_length++; + } + if (flags & HA_CREATE_DELAY_KEY_WRITE) + options|= HA_OPTION_DELAY_KEY_WRITE; + + packed=(packed+7)/8; + if (pack_reclength != INT_MAX32) + pack_reclength+= reclength+packed + + test(test_all_bits(options, HA_OPTION_CHECKSUM | HA_PACK_RECORD)); + min_pack_length+=packed; + + if (!ci->data_file_length) + { + if (ci->max_rows == 0 || pack_reclength == INT_MAX32) + ci->data_file_length= INT_MAX32-1; /* Should be enough */ + else if ((~(ulonglong) 0)/ci->max_rows < (ulonglong) pack_reclength) + ci->data_file_length= ~(ulonglong) 0; + else + ci->data_file_length=(ulonglong) ci->max_rows*pack_reclength; + } + else if (!ci->max_rows) + ci->max_rows=(ha_rows) (ci->data_file_length/(min_pack_length + + ((options & HA_OPTION_PACK_RECORD) ? + 3 : 0))); + + if (options & (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD)) + pointer=mi_get_pointer_length(ci->data_file_length,4); + else + pointer=mi_get_pointer_length(ci->max_rows,4); + if (!(max_rows=(ulonglong) ci->max_rows)) + max_rows= ((((ulonglong) 1 << (pointer*8)) -1) / min_pack_length); + + + real_reclength=reclength; + if (!(options & (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD))) + { + if (reclength <= pointer) + reclength=pointer+1; /* reserve place for delete link */ + } + else + reclength+=long_varchar_count; /* We need space for this! */ + + max_key_length=0; tot_length=0 ; key_segs=0; + max_key_block_length=0; + share.state.rec_per_key_part=rec_per_key_part; + bzero((char*) rec_per_key_part,sizeof(rec_per_key_part)); + share.state.key_root=key_root; + share.state.key_del=key_del; + if (uniques) + { + max_key_block_length= MI_KEY_BLOCK_LENGTH; + max_key_length= MI_UNIQUE_HASH_LENGTH; + } + + for (i=0, keydef=keydefs ; i < keys ; i++ , keydef++) + { + share.state.key_root[i]= HA_OFFSET_ERROR; + + min_key_length_skipp=length=0; + key_length=pointer; + + if (keydef->flag & HA_FULLTEXT) /* SerG */ + { + keydef->flag=HA_FULLTEXT | HA_PACK_KEY | HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + + if (flags & HA_DONT_TOUCH_DATA) + { + /* called by myisamchk - i.e. table structure was taken from + MYI file and FULLTEXT key *do has* additional FT_SEGS keysegs. + We'd better delete them now + */ + keydef->keysegs-=FT_SEGS; + } + + for (j=0, keyseg=keydef->seg ; (int) j < keydef->keysegs ; + j++, keyseg++) + { + if (keyseg->type != HA_KEYTYPE_TEXT && + keyseg->type != HA_KEYTYPE_VARTEXT) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err; + } + } + keydef->keysegs+=FT_SEGS; + + key_length+= HA_FT_MAXLEN+HA_FT_WLEN; +#ifdef EVAL_RUN + key_length++; +#endif + + length++; /* At least one length byte */ + min_key_length_skipp+=HA_FT_MAXLEN; +#if HA_FT_MAXLEN >= 255 + min_key_length_skipp+=2; /* prefix may be 3 bytes */ + length+=2; +#endif + } + else + { + /* Test if prefix compression */ + if (keydef->flag & HA_PACK_KEY) + { + /* Can't use space_compression on number keys */ + if ((keydef->seg[0].flag & HA_SPACE_PACK) && + keydef->seg[0].type == (int) HA_KEYTYPE_NUM) + keydef->seg[0].flag&= ~HA_SPACE_PACK; + + /* Only use HA_PACK_KEY if the first segment is a variable length key */ + if (!(keydef->seg[0].flag & (HA_SPACE_PACK | HA_BLOB_PART | + HA_VAR_LENGTH))) + { + /* pack relative to previous key */ + keydef->flag&= ~HA_PACK_KEY; + keydef->flag|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; + } + else + { + keydef->seg[0].flag|=HA_PACK_KEY; /* for easyer intern test */ + keydef->flag|=HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + } + } + if (keydef->flag & HA_BINARY_PACK_KEY) + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + + if (keydef->flag & HA_AUTO_KEY) + share.base.auto_key=i+1; + for (j=0, keyseg=keydef->seg ; j < keydef->keysegs ; j++, keyseg++) + { + /* numbers are stored with high by first to make compression easier */ + switch (keyseg->type) { + case HA_KEYTYPE_SHORT_INT: + case HA_KEYTYPE_LONG_INT: + case HA_KEYTYPE_FLOAT: + case HA_KEYTYPE_DOUBLE: + case HA_KEYTYPE_USHORT_INT: + case HA_KEYTYPE_ULONG_INT: + case HA_KEYTYPE_LONGLONG: + case HA_KEYTYPE_ULONGLONG: + case HA_KEYTYPE_INT24: + case HA_KEYTYPE_UINT24: + case HA_KEYTYPE_INT8: + keyseg->flag|= HA_SWAP_KEY; + /* fall through */ + default: + break; + } + if (keyseg->flag & HA_SPACE_PACK) + { + keydef->flag |= HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + length++; /* At least one length byte */ + min_key_length_skipp+=keyseg->length; + if (keyseg->length >= 255) + { /* prefix may be 3 bytes */ + min_key_length_skipp+=2; + length+=2; + } + } + if (keyseg->flag & (HA_VAR_LENGTH | HA_BLOB_PART)) + { + keydef->flag|=HA_VAR_LENGTH_KEY; + length++; /* At least one length byte */ + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + min_key_length_skipp+=keyseg->length; + if (keyseg->length >= 255) + { /* prefix may be 3 bytes */ + min_key_length_skipp+=2; + length+=2; + } + } + key_length+= keyseg->length; + if (keyseg->null_bit) + { + key_length++; + options|=HA_OPTION_PACK_KEYS; + keyseg->flag|=HA_NULL_PART; + keydef->flag|=HA_VAR_LENGTH_KEY | HA_NULL_PART_KEY; + } + } + } /* if HA_FULLTEXT */ + key_segs+=keydef->keysegs; + if (keydef->keysegs > MI_MAX_KEY_SEG) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err; + } + if ((keydef->flag & (HA_NOSAME | HA_NULL_PART_KEY)) == HA_NOSAME) + share.state.rec_per_key_part[key_segs-1]=1L; + length+=key_length; + keydef->block_length= MI_BLOCK_SIZE(length,pointer,MI_MAX_KEYPTR_SIZE); + if (keydef->block_length/MI_KEY_BLOCK_LENGTH > MI_MAX_KEY_BLOCK_SIZE) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err; + } + set_if_bigger(max_key_block_length,keydef->block_length); + keydef->keylength= (uint16) key_length; + keydef->minlength= (uint16) (length-min_key_length_skipp); + keydef->maxlength= (uint16) length; + + if (length > max_key_length) + max_key_length= length; + tot_length+= (max_rows/(ulong) (((uint) keydef->block_length-5)/ + (length*2)))* + (ulong) keydef->block_length; + } + for (i=max_key_block_length/MI_KEY_BLOCK_LENGTH ; i-- ; ) + key_del[i]=HA_OFFSET_ERROR; + + unique_key_parts=0; + offset=reclength-uniques*MI_UNIQUE_HASH_LENGTH; + for (i=0, uniquedef=uniquedefs ; i < uniques ; i++ , uniquedef++) + { + uniquedef->key=keys+i; + unique_key_parts+=uniquedef->keysegs; + share.state.key_root[keys+i]= HA_OFFSET_ERROR; + } + keys+=uniques; /* Each unique has 1 key */ + key_segs+=uniques; /* Each unique has 1 key seg */ + + base_pos=(MI_STATE_INFO_SIZE + keys * MI_STATE_KEY_SIZE + + max_key_block_length/MI_KEY_BLOCK_LENGTH*MI_STATE_KEYBLOCK_SIZE+ + key_segs*MI_STATE_KEYSEG_SIZE); + info_length=base_pos+(uint) (MI_BASE_INFO_SIZE+ + keys * MI_KEYDEF_SIZE+ + uniques * MI_UNIQUEDEF_SIZE + + (key_segs + unique_key_parts)*MI_KEYSEG_SIZE+ + columns*MI_COLUMNDEF_SIZE); + + bmove(share.state.header.file_version,(byte*) myisam_file_magic,4); + ci->old_options=options| (ci->old_options & HA_OPTION_TEMP_COMPRESS_RECORD ? + HA_OPTION_COMPRESS_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD: 0); + mi_int2store(share.state.header.options,ci->old_options); + mi_int2store(share.state.header.header_length,info_length); + mi_int2store(share.state.header.state_info_length,MI_STATE_INFO_SIZE); + mi_int2store(share.state.header.base_info_length,MI_BASE_INFO_SIZE); + mi_int2store(share.state.header.base_pos,base_pos); + share.state.header.language= (ci->language ? + ci->language : MY_CHARSET_CURRENT); + share.state.header.max_block_size=max_key_block_length/MI_KEY_BLOCK_LENGTH; + + share.state.dellink = HA_OFFSET_ERROR; + share.state.process= (ulong) getpid(); + share.state.unique= (ulong) 0; + share.state.version= (ulong) time((time_t*) 0); + share.state.sortkey= (ushort) ~0; + share.state.auto_increment=ci->auto_increment; + share.options=options; + share.base.rec_reflength=pointer; + share.base.key_reflength= + mi_get_pointer_length((tot_length + max_key_block_length * keys * + MI_INDEX_BLOCK_MARGIN) / MI_KEY_BLOCK_LENGTH, + 3); + share.base.keys= share.state.header.keys = keys; + share.state.header.uniques= uniques; + mi_int2store(share.state.header.key_parts,key_segs); + mi_int2store(share.state.header.unique_key_parts,unique_key_parts); + + share.state.key_map = ((ulonglong) 1 << keys)-1; + share.base.keystart = share.state.state.key_file_length= + MY_ALIGN(info_length, myisam_block_size); + share.base.max_key_block_length=max_key_block_length; + share.base.max_key_length=ALIGN_SIZE(max_key_length+4); + share.base.records=ci->max_rows; + share.base.reloc= ci->reloc_rows; + share.base.reclength=real_reclength; + share.base.pack_reclength=reclength+ test(options & HA_OPTION_CHECKSUM);; + share.base.max_pack_length=pack_reclength; + share.base.min_pack_length=min_pack_length; + share.base.pack_bits=packed; + share.base.fields=fields; + share.base.pack_fields=packed; +#ifdef USE_RAID + share.base.raid_type=ci->raid_type; + share.base.raid_chunks=ci->raid_chunks; + share.base.raid_chunksize=ci->raid_chunksize; +#endif + + /* max_data_file_length and max_key_file_length are recalculated on open */ + if (options & HA_OPTION_TMP_TABLE) + share.base.max_data_file_length=(my_off_t) ci->data_file_length; + + share.base.min_block_length= + (share.base.pack_reclength+3 < MI_EXTEND_BLOCK_LENGTH && + ! share.base.blobs) ? + max(share.base.pack_reclength,MI_MIN_BLOCK_LENGTH) : + MI_EXTEND_BLOCK_LENGTH; + if (! (flags & HA_DONT_TOUCH_DATA)) + share.state.create_time= (long) time((time_t*) 0); + + if ((file = my_create(fn_format(buff,name,"",MI_NAME_IEXT,4),0, + O_RDWR | O_TRUNC,MYF(MY_WME))) < 0) + goto err; + errpos=1; + VOID(fn_format(buff,name,"",MI_NAME_DEXT,2+4)); + if (!(flags & HA_DONT_TOUCH_DATA)) + { +#ifdef USE_RAID + if (share.base.raid_type) + { + if ((dfile=my_raid_create(buff,0,O_RDWR | O_TRUNC, + share.base.raid_type, + share.base.raid_chunks, + share.base.raid_chunksize, + MYF(MY_WME | MY_RAID))) < 0) + goto err; + } + else +#endif + if ((dfile = my_create(buff,0,O_RDWR | O_TRUNC,MYF(MY_WME))) < 0) + goto err; + + errpos=3; + } + + if (mi_state_info_write(file, &share.state, 2) || + mi_base_info_write(file, &share.base)) + goto err; +#ifndef DBUG_OFF + if ((uint) my_tell(file,MYF(0)) != base_pos+ MI_BASE_INFO_SIZE) + { + uint pos=(uint) my_tell(file,MYF(0)); + DBUG_PRINT("warning",("base_length: %d != used_length: %d", + base_pos+ MI_BASE_INFO_SIZE, pos)); + } +#endif + + /* Write key and keyseg definitions */ + for (i=0 ; i < share.base.keys - uniques; i++) + { + uint ft_segs=(keydefs[i].flag & HA_FULLTEXT) ? FT_SEGS : 0; /* SerG */ + + if (mi_keydef_write(file, &keydefs[i])) + goto err; + for (j=0 ; j < keydefs[i].keysegs-ft_segs ; j++) + if (mi_keyseg_write(file, &keydefs[i].seg[j])) + goto err; + for (j=0 ; j < ft_segs ; j++) /* SerG */ + if (mi_keyseg_write(file, &ft_keysegs[j])) + goto err; + } + /* Create extra keys for unique definitions */ + offset=reclength-uniques*MI_UNIQUE_HASH_LENGTH; + bzero((char*) &tmp_keydef,sizeof(tmp_keydef)); + bzero((char*) &tmp_keyseg,sizeof(tmp_keyseg)); + for (i=0; i < uniques ; i++) + { + tmp_keydef.keysegs=1; + tmp_keydef.flag= HA_UNIQUE_CHECK; + tmp_keydef.block_length= MI_KEY_BLOCK_LENGTH; + tmp_keydef.keylength= MI_UNIQUE_HASH_LENGTH + pointer; + tmp_keydef.minlength=tmp_keydef.maxlength=tmp_keydef.keylength; + tmp_keyseg.type= MI_UNIQUE_HASH_TYPE; + tmp_keyseg.length= MI_UNIQUE_HASH_LENGTH; + tmp_keyseg.start= offset; + offset+= MI_UNIQUE_HASH_LENGTH; + if (mi_keydef_write(file,&tmp_keydef) || + mi_keyseg_write(file,(&tmp_keyseg))) + goto err; + } + + /* Save unique definition */ + for (i=0 ; i < share.state.header.uniques ; i++) + { + if (mi_uniquedef_write(file, &uniquedefs[i])) + goto err; + for (j=0 ; j < uniquedefs[i].keysegs ; j++) + { + if (mi_keyseg_write(file, &uniquedefs[i].seg[j])) + goto err; + } + } + for (i=0 ; i < share.base.fields ; i++) + if (mi_recinfo_write(file, &recinfo[i])) + goto err; + +#ifndef DBUG_OFF + if ((uint) my_tell(file,MYF(0)) != info_length) + { + uint pos= (uint) my_tell(file,MYF(0)); + DBUG_PRINT("warning",("info_length: %d != used_length: %d", + info_length, pos)); + } +#endif + + /* Enlarge files */ + if (my_chsize(file,(ulong) share.base.keystart,MYF(0))) + goto err; + + if (! (flags & HA_DONT_TOUCH_DATA)) + { +#ifdef USE_RELOC + if (my_chsize(dfile,share.base.min_pack_length*ci->reloc_rows,MYF(0))) + goto err; +#endif + errpos=2; + if (my_close(dfile,MYF(0))) + goto err; + } + errpos=0; + pthread_mutex_unlock(&THR_LOCK_myisam); + if (my_close(file,MYF(0))) + goto err; + DBUG_RETURN(0); + +err: + pthread_mutex_unlock(&THR_LOCK_myisam); + save_errno=my_errno; + switch (errpos) { + case 3: + VOID(my_close(dfile,MYF(0))); + /* fall through */ + case 2: + if (! (flags & HA_DONT_TOUCH_DATA)) + { + /* QQ: Tõnu should add a call to my_raid_delete() here */ + VOID(fn_format(buff,name,"",MI_NAME_DEXT,2+4)); + my_delete(buff,MYF(0)); + } + /* fall through */ + case 1: + VOID(my_close(file,MYF(0))); + if (! (flags & HA_DONT_TOUCH_DATA)) + { + VOID(fn_format(buff,name,"",MI_NAME_IEXT,2+4)); + my_delete(buff,MYF(0)); + } + } + DBUG_RETURN(my_errno=save_errno); /* return the fatal errno */ +} + + +uint mi_get_pointer_length(ulonglong file_length, uint def) +{ + if (file_length) /* If not default */ + { + if (file_length >= (longlong) 1 << 56) + def=8; + if (file_length >= (longlong) 1 << 48) + def=7; + if (file_length >= (longlong) 1 << 40) + def=6; + else if (file_length >= (longlong) 1 << 32) + def=5; + else if (file_length >= (1L << 24)) + def=4; + else if (file_length >= (1L << 16)) + def=3; + else + def=2; + } + return def; +} diff --git a/myisam/mi_dbug.c b/myisam/mi_dbug.c new file mode 100644 index 00000000000..62d6e039059 --- /dev/null +++ b/myisam/mi_dbug.c @@ -0,0 +1,149 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Support rutiner with are using with dbug */ + +#include "myisamdef.h" + + /* Print a key in user understandable format */ + +void _mi_print_key(FILE *stream, register MI_KEYSEG *keyseg, + const uchar *key, uint length) +{ + int flag; + short int s_1; + long int l_1; + float f_1; + double d_1; + const uchar *end; + const uchar *key_end=key+length; + + VOID(fputs("Key: \"",stream)); + flag=0; + for (; keyseg->type && key < key_end ;keyseg++) + { + if (flag++) + VOID(putc('-',stream)); + end= key+ keyseg->length; + if (keyseg->flag & HA_NULL_PART) + { + if (!*key) + { + fprintf(stream,"NULL"); + continue; + } + key++; + } + + switch (keyseg->type) { + case HA_KEYTYPE_BINARY: + if (!(keyseg->flag & HA_SPACE_PACK) && keyseg->length == 1) + { /* packed binary digit */ + VOID(fprintf(stream,"%d",(uint) *key++)); + break; + } + /* fall through */ + case HA_KEYTYPE_TEXT: + case HA_KEYTYPE_NUM: + if (keyseg->flag & HA_SPACE_PACK) + { + VOID(fprintf(stream,"%.*s",(int) *key,key+1)); + key+= (int) *key+1; + } + else + { + VOID(fprintf(stream,"%.*s",(int) keyseg->length,key)); + key=end; + } + break; + case HA_KEYTYPE_INT8: + VOID(fprintf(stream,"%d",(int) *((signed char*) key))); + key=end; + break; + case HA_KEYTYPE_SHORT_INT: + s_1= mi_sint2korr(key); + VOID(fprintf(stream,"%d",(int) s_1)); + key=end; + break; + case HA_KEYTYPE_USHORT_INT: + { + ushort u_1; + u_1= mi_uint2korr(key); + VOID(fprintf(stream,"%u",(uint) u_1)); + key=end; + break; + } + case HA_KEYTYPE_LONG_INT: + l_1=mi_sint4korr(key); + VOID(fprintf(stream,"%ld",l_1)); + key=end; + break; + case HA_KEYTYPE_ULONG_INT: + l_1=mi_sint4korr(key); + VOID(fprintf(stream,"%lu",(ulong) l_1)); + key=end; + break; + case HA_KEYTYPE_INT24: + VOID(fprintf(stream,"%ld",(long) mi_sint3korr(key))); + key=end; + break; + case HA_KEYTYPE_UINT24: + VOID(fprintf(stream,"%lu",(ulong) mi_uint3korr(key))); + key=end; + break; + case HA_KEYTYPE_FLOAT: + mi_float4get(f_1,key); + VOID(fprintf(stream,"%g",(double) f_1)); + key=end; + break; + case HA_KEYTYPE_DOUBLE: + mi_float8get(d_1,key); + VOID(fprintf(stream,"%g",d_1)); + key=end; + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + { + char buff[21]; + longlong2str(mi_sint8korr(key),buff,-10); + VOID(fprintf(stream,"%s",buff)); + key=end; + break; + } + case HA_KEYTYPE_ULONGLONG: + { + char buff[21]; + longlong2str(mi_sint8korr(key),buff,10); + VOID(fprintf(stream,"%s",buff)); + key=end; + break; + } +#endif + case HA_KEYTYPE_VARTEXT: /* VARCHAR and TEXT */ + case HA_KEYTYPE_VARBINARY: /* VARBINARY and BLOB */ + { + uint tmp_length; + get_key_length(tmp_length,key); + VOID(fprintf(stream,"%.*s",(int) tmp_length,key)); + key+=tmp_length; + break; + } + default: break; /* This never happens */ + } + } + VOID(fputs("\"\n",stream)); + return; +} /* print_key */ diff --git a/myisam/mi_delete.c b/myisam/mi_delete.c new file mode 100644 index 00000000000..4c5f3c79e17 --- /dev/null +++ b/myisam/mi_delete.c @@ -0,0 +1,780 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Remove a row from a MyISAM table */ + +#include "fulltext.h" +#ifdef __WIN__ +#include <errno.h> +#endif + +static int d_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uint key_length, my_off_t page, uchar *anc_buff); +static int del(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,uchar *anc_buff, + my_off_t leaf_page,uchar *leaf_buff,uchar *keypos, + my_off_t next_block,uchar *ret_key); +static int underflow(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *anc_buff, + my_off_t leaf_page, uchar *leaf_buff,uchar *keypos); +static uint remove_key(MI_KEYDEF *keyinfo,uint nod_flag,uchar *keypos, + uchar *lastkey,uchar *page_end, + my_off_t *next_block); + + +int mi_delete(MI_INFO *info,const byte *record) +{ + uint i; + uchar *old_key; + int save_errno; + char lastpos[8]; + + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_delete"); + + /* Test if record is in datafile */ + + if (!(info->update & HA_STATE_AKTIV)) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); /* No database read */ + } + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + if (info->s->calc_checksum) + info->checksum=(*info->s->calc_checksum)(info,record); + if ((*share->compare_record)(info,record)) + goto err; /* Error on read-check */ + + if (_mi_mark_file_changed(info)) + goto err; + + /* Remove all keys from the .ISAM file */ + + old_key=info->lastkey2; + for (i=0 ; i < share->base.keys ; i++ ) + { + if (((ulonglong) 1 << i) & info->s->state.key_map) + { + info->s->keyinfo[i].version++; + /* The following code block is for text searching by SerG */ + if (info->s->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_del(info,i,(char*) old_key,record,info->lastpos)) + goto err; + } + else + { + uint key_length=_mi_make_key(info,i,old_key,record,info->lastpos); + if (_mi_ck_delete(info,i,old_key,key_length)) + goto err; + } + } + } + + if ((*share->delete_record)(info)) + goto err; /* Remove record from database */ + info->s->state.checksum-=info->checksum; + + info->update= HA_STATE_CHANGED+HA_STATE_DELETED+HA_STATE_ROW_CHANGED; + info->state->records--; + + mi_sizestore(lastpos,info->lastpos); + myisam_log_command(MI_LOG_DELETE,info,(byte*) lastpos,sizeof(lastpos),0); + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(0); + +err: + save_errno=my_errno; + mi_sizestore(lastpos,info->lastpos); + myisam_log_command(MI_LOG_DELETE,info,(byte*) lastpos, sizeof(lastpos),0); + if (save_errno != HA_ERR_RECORD_CHANGED) + mi_mark_crashed(info); /* mark table crashed */ + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + info->update|=HA_STATE_WRITTEN; /* Buffer changed */ + allow_break(); /* Allow SIGHUP & SIGINT */ + my_errno=save_errno; + if (save_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_CRASHED; + + DBUG_RETURN(my_errno); +} /* mi_delete */ + + + /* Remove a key from the btree index */ + +int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + uint nod_flag; + my_off_t old_root; + uchar *root_buff; + MI_KEYDEF *keyinfo; + DBUG_ENTER("_mi_ck_delete"); + + if ((old_root=info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + DBUG_RETURN(my_errno=HA_ERR_CRASHED); + } + keyinfo=info->s->keyinfo+keynr; + if (!(root_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + MI_MAX_KEY_BUFF*2))) + { + DBUG_PRINT("error",("Couldn't allocate memory")); + DBUG_RETURN(my_errno=ENOMEM); + } + DBUG_PRINT("info",("root_page: %ld",old_root)); + if (!_mi_fetch_keypage(info,keyinfo,old_root,root_buff,0)) + { + error= -1; + goto err; + } + if ((error=d_search(info,keyinfo,key,key_length,old_root,root_buff)) >0) + { + if (error == 2) + { + DBUG_PRINT("test",("Enlarging of root when deleting")); + error=_mi_enlarge_root(info,keynr,key); + } + else /* error == 1 */ + { + if (mi_getint(root_buff) <= (nod_flag=mi_test_if_nod(root_buff))+3) + { + error=0; + if (nod_flag) + info->s->state.key_root[keynr]=_mi_kpos(nod_flag, + root_buff+2+nod_flag); + else + info->s->state.key_root[keynr]= HA_OFFSET_ERROR; + if (_mi_dispose(info,keyinfo,old_root)) + error= -1; + } + else + error=_mi_write_keypage(info,keyinfo,old_root,root_buff); + } + } +err: + my_afree((gptr) root_buff); + DBUG_RETURN(error); +} /* _mi_ck_delete */ + + + /* + ** Remove key below key root + ** Return values: + ** 1 if there are less buffers; In this case anc_buff is not saved + ** 2 if there are more buffers + ** -1 on errors + */ + +static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_length, my_off_t page, + uchar *anc_buff) +{ + int flag,ret_value,save_flag; + uint length,nod_flag; + my_bool last_key; + uchar *leaf_buff,*keypos; + my_off_t leaf_page,next_block; + uchar lastkey[MI_MAX_KEY_BUFF]; + DBUG_ENTER("d_search"); + DBUG_DUMP("page",(byte*) anc_buff,mi_getint(anc_buff)); + + flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,key_length,SEARCH_SAME, + &keypos, lastkey, &last_key); + if (flag == MI_FOUND_WRONG_KEY) + { + DBUG_PRINT("error",("Found wrong key")); + DBUG_RETURN(-1); + } + nod_flag=mi_test_if_nod(anc_buff); + + leaf_buff=0; + LINT_INIT(leaf_page); + if (nod_flag) + { + leaf_page=_mi_kpos(nod_flag,keypos); + if (!(leaf_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + MI_MAX_KEY_BUFF*2))) + { + DBUG_PRINT("error",("Couldn't allocate memory")); + my_errno=ENOMEM; + DBUG_RETURN(-1); + } + if (!_mi_fetch_keypage(info,keyinfo,leaf_page,leaf_buff,0)) + goto err; + } + + if (flag != 0) + { + if (!nod_flag) + { + DBUG_PRINT("error",("Didn't find key")); + my_errno=HA_ERR_CRASHED; /* This should newer happend */ + goto err; + } + save_flag=0; + ret_value=d_search(info,keyinfo,key,key_length,leaf_page,leaf_buff); + } + else + { /* Found key */ + uint tmp; + length=mi_getint(anc_buff); + tmp=remove_key(keyinfo,nod_flag,keypos,lastkey,anc_buff+length, + &next_block); + if (tmp == 0) + DBUG_RETURN(0); + length-= tmp; + + mi_putint(anc_buff,length,nod_flag); + if (!nod_flag) + { /* On leaf page */ + if (_mi_write_keypage(info,keyinfo,page,anc_buff)) + DBUG_RETURN(-1); + if (length <= (uint) keyinfo->underflow_block_length) + DBUG_RETURN(1); /* Page will be update later */ + DBUG_RETURN(0); + } + save_flag=1; + ret_value=del(info,keyinfo,key,anc_buff,leaf_page,leaf_buff,keypos, + next_block,lastkey); + } + if (ret_value >0) + { + save_flag=1; + if (ret_value == 1) + ret_value= underflow(info,keyinfo,anc_buff,leaf_page,leaf_buff,keypos); + else + { /* This happens only with packed keys */ + DBUG_PRINT("test",("Enlarging of key when deleting")); + if (!_mi_get_last_key(info,keyinfo,anc_buff,lastkey,keypos,&length)) + { + goto err; + } + ret_value=_mi_insert(info,keyinfo,key,anc_buff,keypos,lastkey, + (uchar*) 0,(uchar*) 0,(my_off_t) 0,(my_bool) 0); + } + } + if (ret_value == 0 && mi_getint(anc_buff) > keyinfo->block_length) + { + save_flag=1; + ret_value=_mi_split_page(info,keyinfo,key,anc_buff,lastkey,0) | 2; + } + if (save_flag && ret_value != 1) + ret_value|=_mi_write_keypage(info,keyinfo,page,anc_buff); + else + { + DBUG_DUMP("page",(byte*) anc_buff,mi_getint(anc_buff)); + } + my_afree((byte*) leaf_buff); + DBUG_RETURN(ret_value); +err: + my_afree((byte*) leaf_buff); + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1); +} /* d_search */ + + + /* Remove a key that has a page-reference */ + +static int del(register MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *key, + uchar *anc_buff, my_off_t leaf_page, uchar *leaf_buff, + uchar *keypos, /* Pos to where deleted key was */ + my_off_t next_block, + uchar *ret_key) /* key before keypos in anc_buff */ +{ + int ret_value,length; + uint a_length,nod_flag,tmp; + my_off_t next_page; + uchar keybuff[MI_MAX_KEY_BUFF],*endpos,*next_buff,*key_start, *prev_key; + MYISAM_SHARE *share=info->s; + MI_KEY_PARAM s_temp; + DBUG_ENTER("del"); + DBUG_PRINT("enter",("leaf_page: %ld keypos: %lx",leaf_page,keypos)); + DBUG_DUMP("leaf_buff",(byte*) leaf_buff,mi_getint(leaf_buff)); + + endpos=leaf_buff+mi_getint(leaf_buff); + if (!(key_start=_mi_get_last_key(info,keyinfo,leaf_buff,keybuff,endpos, + &tmp))) + DBUG_RETURN(-1); + + if ((nod_flag=mi_test_if_nod(leaf_buff))) + { + next_page= _mi_kpos(nod_flag,endpos); + if (!(next_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + MI_MAX_KEY_BUFF*2))) + DBUG_RETURN(-1); + if (!_mi_fetch_keypage(info,keyinfo,next_page,next_buff,0)) + ret_value= -1; + else + { + DBUG_DUMP("next_page",(byte*) next_buff,mi_getint(next_buff)); + if ((ret_value=del(info,keyinfo,key,anc_buff,next_page,next_buff, + keypos,next_block,ret_key)) >0) + { + endpos=leaf_buff+mi_getint(leaf_buff); + if (ret_value == 1) + { + ret_value=underflow(info,keyinfo,leaf_buff,next_page, + next_buff,endpos); + if (ret_value == 0 && mi_getint(leaf_buff) > keyinfo->block_length) + { + ret_value=_mi_split_page(info,keyinfo,key,leaf_buff,ret_key,0) | 2; + } + } + else + { + DBUG_PRINT("test",("Inserting of key when deleting")); + if (_mi_get_last_key(info,keyinfo,leaf_buff,keybuff,endpos, + &tmp)) + goto err; + ret_value=_mi_insert(info,keyinfo,key,leaf_buff,endpos,keybuff, + (uchar*) 0,(uchar*) 0,(my_off_t) 0,0); + } + } + if (_mi_write_keypage(info,keyinfo,leaf_page,leaf_buff)) + goto err; + } + my_afree((byte*) next_buff); + DBUG_RETURN(ret_value); + } + + /* Remove last key from leaf page */ + + mi_putint(leaf_buff,key_start-leaf_buff,nod_flag); + if (_mi_write_keypage(info,keyinfo,leaf_page,leaf_buff)) + goto err; + + /* Place last key in ancestor page on deleted key position */ + + a_length=mi_getint(anc_buff); + endpos=anc_buff+a_length; + if (keypos != anc_buff+2+share->base.key_reflength && + !_mi_get_last_key(info,keyinfo,anc_buff,ret_key,keypos,&tmp)) + goto err; + prev_key=(keypos == anc_buff+2+share->base.key_reflength ? + 0 : ret_key); + length=(*keyinfo->pack_key)(keyinfo,share->base.key_reflength, + keypos == endpos ? (uchar*) 0 : keypos, + prev_key, prev_key, + keybuff,&s_temp); + if (length > 0) + bmove_upp((byte*) endpos+length,(byte*) endpos,(uint) (endpos-keypos)); + else + bmove(keypos,keypos-length, (int) (endpos-keypos)+length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + /* Save pointer to next leaf */ + if (!(*keyinfo->get_key)(keyinfo,share->base.key_reflength,&keypos,ret_key)) + goto err; + _mi_kpointer(info,keypos - share->base.key_reflength,next_block); + mi_putint(anc_buff,a_length+length,share->base.key_reflength); + + DBUG_RETURN( mi_getint(leaf_buff) <= (uint) keyinfo->underflow_block_length); +err: + DBUG_RETURN(-1); +} /* del */ + + + /* Balances adjacent pages if underflow occours */ + +static int underflow(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *anc_buff, + my_off_t leaf_page,/* Ancestor page and underflow page */ + uchar *leaf_buff, + uchar *keypos) /* Position to pos after key */ +{ + int t_length; + uint length,anc_length,buff_length,leaf_length,p_length,s_length,nod_flag, + key_reflength,key_length; + my_off_t next_page; + uchar anc_key[MI_MAX_KEY_BUFF],leaf_key[MI_MAX_KEY_BUFF], + *buff,*endpos,*next_keypos,*anc_pos,*half_pos,*temp_pos,*prev_key, + *after_key; + MI_KEY_PARAM s_temp; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("underflow"); + DBUG_PRINT("enter",("leaf_page: %ld keypos: %lx",(long) leaf_page,keypos)); + DBUG_DUMP("anc_buff",(byte*) anc_buff,mi_getint(anc_buff)); + DBUG_DUMP("leaf_buff",(byte*) leaf_buff,mi_getint(leaf_buff)); + + buff=info->buff; + info->buff_used=1; + next_keypos=keypos; + nod_flag=mi_test_if_nod(leaf_buff); + p_length=nod_flag+2; + anc_length=mi_getint(anc_buff); + leaf_length=mi_getint(leaf_buff); + key_reflength=share->base.key_reflength; + if (info->s->keyinfo+info->lastinx == keyinfo) + info->page_changed=1; + + if ((keypos < anc_buff+anc_length && (share->rnd++ & 1)) || + keypos == anc_buff+2+key_reflength) + { /* Use page right of anc-page */ + DBUG_PRINT("test",("use right page")); + + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { + if (!(next_keypos=_mi_get_key(info, keyinfo, + anc_buff, buff, keypos, &length))) + goto err; + } + else + { + /* Got to end of found key */ + buff[0]=buff[1]=0; /* Avoid length error check if packed key */ + if (!(*keyinfo->get_key)(keyinfo,key_reflength,&next_keypos, + buff)) + goto err; + } + next_page= _mi_kpos(key_reflength,next_keypos); + if (!_mi_fetch_keypage(info,keyinfo,next_page,buff,0)) + goto err; + buff_length=mi_getint(buff); + DBUG_DUMP("next",(byte*) buff,buff_length); + + /* find keys to make a big key-page */ + bmove((byte*) next_keypos-key_reflength,(byte*) buff+2, + key_reflength); + if (!_mi_get_last_key(info,keyinfo,anc_buff,anc_key,next_keypos,&length) + || !_mi_get_last_key(info,keyinfo,leaf_buff,leaf_key, + leaf_buff+leaf_length,&length)) + goto err; + + /* merge pages and put parting key from anc_buff between */ + prev_key=(leaf_length == p_length ? (uchar*) 0 : leaf_key); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,buff+p_length, + prev_key, prev_key, + anc_key, &s_temp); + length=buff_length-p_length; + endpos=buff+length+leaf_length+t_length; + /* buff will always be larger than before !*/ + bmove_upp((byte*) endpos, (byte*) buff+buff_length,length); + memcpy((byte*) buff, (byte*) leaf_buff,(size_t) leaf_length); + (*keyinfo->store_key)(keyinfo,buff+leaf_length,&s_temp); + buff_length=(uint) (endpos-buff); + mi_putint(buff,buff_length,nod_flag); + + /* remove key from anc_buff */ + + s_length=remove_key(keyinfo,key_reflength,keypos,anc_key, + anc_buff+anc_length,(my_off_t *) 0); + if (!s_length) + goto err; + anc_length-=s_length; + mi_putint(anc_buff,anc_length,key_reflength); + + if (buff_length <= keyinfo->block_length) + { /* Keys in one page */ + memcpy((byte*) leaf_buff,(byte*) buff,(size_t) buff_length); + if (_mi_dispose(info,keyinfo,next_page)) + goto err; + } + else + { /* Page is full */ + endpos=anc_buff+anc_length; + DBUG_PRINT("test",("anc_buff: %lx endpos: %lx",anc_buff,endpos)); + if (keypos != anc_buff+2+key_reflength && + !_mi_get_last_key(info,keyinfo,anc_buff,anc_key,keypos,&length)) + goto err; + if (!(half_pos=_mi_find_half_pos(nod_flag, keyinfo, buff, leaf_key, + &key_length, &after_key))) + goto err; + length=(uint) (half_pos-buff); + memcpy((byte*) leaf_buff,(byte*) buff,(size_t) length); + mi_putint(leaf_buff,length,nod_flag); + + /* Correct new keypointer to leaf_page */ + half_pos=after_key; + _mi_kpointer(info,leaf_key+key_length,next_page); + /* Save key in anc_buff */ + prev_key=(keypos == anc_buff+2+key_reflength ? (uchar*) 0 : anc_key), + t_length=(*keyinfo->pack_key)(keyinfo,key_reflength, + (keypos == endpos ? (uchar*) 0 : + keypos), + prev_key, prev_key, + leaf_key, &s_temp); + if (t_length >= 0) + bmove_upp((byte*) endpos+t_length,(byte*) endpos, + (uint) (endpos-keypos)); + else + bmove(keypos,keypos-t_length,(uint) (endpos-keypos)+t_length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + mi_putint(anc_buff,(anc_length+=t_length),key_reflength); + + /* Store key first in new page */ + if (nod_flag) + bmove((byte*) buff+2,(byte*) half_pos-nod_flag,(size_t) nod_flag); + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&half_pos,leaf_key)) + goto err; + t_length=(int) (*keyinfo->pack_key)(keyinfo, nod_flag, (uchar*) 0, + (uchar*) 0, (uchar *) 0, + leaf_key, &s_temp); + /* t_length will always be > 0 for a new page !*/ + length=(buff+mi_getint(buff))-half_pos; + bmove((byte*) buff+p_length+t_length,(byte*) half_pos,(size_t) length); + (*keyinfo->store_key)(keyinfo,buff+p_length,&s_temp); + mi_putint(buff,length+t_length+p_length,nod_flag); + + if (_mi_write_keypage(info,keyinfo,next_page,buff)) + goto err; + } + if (_mi_write_keypage(info,keyinfo,leaf_page,leaf_buff)) + goto err; + DBUG_RETURN(anc_length <= (uint) keyinfo->underflow_block_length); + } + + DBUG_PRINT("test",("use left page")); + + keypos=_mi_get_last_key(info,keyinfo,anc_buff,anc_key,keypos,&length); + if (!keypos) + goto err; + next_page= _mi_kpos(key_reflength,keypos); + if (!_mi_fetch_keypage(info,keyinfo,next_page,buff,0)) + goto err; + buff_length=mi_getint(buff); + endpos=buff+buff_length; + DBUG_DUMP("prev",(byte*) buff,buff_length); + + /* find keys to make a big key-page */ + bmove((byte*) next_keypos - key_reflength,(byte*) leaf_buff+2, + key_reflength); + next_keypos=keypos; + if (!(*keyinfo->get_key)(keyinfo,key_reflength,&next_keypos, + anc_key)) + goto err; + if (!_mi_get_last_key(info,keyinfo,buff,leaf_key,endpos,&length)) + goto err; + + /* merge pages and put parting key from anc_buff between */ + prev_key=(leaf_length == p_length ? (uchar*) 0 : leaf_key); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (leaf_length == p_length ? + (uchar*) 0 : leaf_buff+p_length), + prev_key, prev_key, + anc_key, &s_temp); + if (t_length >= 0) + bmove((byte*) endpos+t_length,(byte*) leaf_buff+p_length, + (size_t) (leaf_length-p_length)); + else /* We gained space */ + bmove((byte*) endpos,(byte*) leaf_buff+((int) p_length-t_length), + (size_t) (leaf_length-p_length+t_length)); + + (*keyinfo->store_key)(keyinfo,endpos,&s_temp); + buff_length=buff_length+leaf_length-p_length+t_length; + mi_putint(buff,buff_length,nod_flag); + + /* remove key from anc_buff */ + s_length=remove_key(keyinfo,key_reflength,keypos,anc_key, + anc_buff+anc_length,(my_off_t *) 0); + if (!s_length) + goto err; + anc_length-=s_length; + mi_putint(anc_buff,anc_length,key_reflength); + + if (buff_length <= keyinfo->block_length) + { /* Keys in one page */ + if (_mi_dispose(info,keyinfo,leaf_page)) + goto err; + } + else + { /* Page is full */ + if (keypos == anc_buff+2+key_reflength) + anc_pos=0; /* First key */ + else if (!_mi_get_last_key(info,keyinfo,anc_buff,anc_pos=anc_key,keypos, + &length)) + goto err; + endpos=_mi_find_half_pos(nod_flag,keyinfo,buff,leaf_key, + &key_length, &half_pos); + if (!endpos) + goto err; + _mi_kpointer(info,leaf_key+key_length,leaf_page); + /* Save key in anc_buff */ + DBUG_DUMP("anc_buff",(byte*) anc_buff,anc_length); + DBUG_DUMP("key_to_anc",(byte*) leaf_key,key_length); + + temp_pos=anc_buff+anc_length; + t_length=(*keyinfo->pack_key)(keyinfo,key_reflength, + keypos == temp_pos ? (uchar*) 0 + : keypos, + anc_pos, anc_pos, + leaf_key,&s_temp); + if (t_length > 0) + bmove_upp((byte*) temp_pos+t_length,(byte*) temp_pos, + (uint) (temp_pos-keypos)); + else + bmove(keypos,keypos-t_length,(uint) (temp_pos-keypos)+t_length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + mi_putint(anc_buff,(anc_length+=t_length),key_reflength); + + /* Store first key on new page */ + if (nod_flag) + bmove((byte*) leaf_buff+2,(byte*) half_pos-nod_flag,(size_t) nod_flag); + if (!(length=(*keyinfo->get_key)(keyinfo,nod_flag,&half_pos,leaf_key))) + goto err; + DBUG_DUMP("key_to_leaf",(byte*) leaf_key,length); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, (uchar*) 0, + (uchar*) 0, (uchar*) 0, leaf_key, &s_temp); + length=(uint) ((buff+buff_length)-half_pos); + DBUG_PRINT("info",("t_length: %d length: %d",t_length,(int) length)); + bmove((byte*) leaf_buff+p_length+t_length,(byte*) half_pos, + (size_t) length); + (*keyinfo->store_key)(keyinfo,leaf_buff+p_length,&s_temp); + mi_putint(leaf_buff,length+t_length+p_length,nod_flag); + if (_mi_write_keypage(info,keyinfo,leaf_page,leaf_buff)) + goto err; + mi_putint(buff,endpos-buff,nod_flag); + } + if (_mi_write_keypage(info,keyinfo,next_page,buff)) + goto err; + DBUG_RETURN(anc_length <= (uint) keyinfo->block_length/2); +err: + DBUG_RETURN(-1); +} /* underflow */ + + + /* + remove a key from packed buffert + The current code doesn't handle the case that the next key may be + packed better against the previous key if there is a case difference + returns how many chars was removed or 0 on error + */ + +static uint remove_key(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *keypos, /* Where key starts */ + uchar *lastkey, /* key to be removed */ + uchar *page_end, /* End of page */ + my_off_t *next_block) /* ptr to next block */ +{ + int s_length; + uchar *start; + DBUG_ENTER("remove_key"); + DBUG_PRINT("enter",("keypos: %lx page_end: %lx",keypos,page_end)); + + start=keypos; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + s_length=(int) (keyinfo->keylength+nod_flag); + if (next_block && nod_flag) + *next_block= _mi_kpos(nod_flag,keypos+s_length); + } + else + { /* Let keypos point at next key */ + /* Calculate length of key */ + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey)) + DBUG_RETURN(0); /* Error */ + if (next_block && nod_flag) + *next_block= _mi_kpos(nod_flag,keypos); + s_length=(keypos-start); + if (keypos != page_end) + { + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { + uchar *old_key=start; + uint next_length,prev_length,prev_pack_length; + get_key_length(next_length,keypos); + get_key_pack_length(prev_length,prev_pack_length,old_key); + if (next_length > prev_length) + { + /* We have to copy data from the current key to the next key */ + bmove_upp((char*) keypos,(char*) (lastkey+next_length), + (next_length-prev_length)); + keypos-=(next_length-prev_length)+prev_pack_length; + store_key_length(keypos,prev_length); + s_length=(keypos-start); + } + } + else + { + /* A variable length first key part */ + if (*keypos & 128) /* Next key is packed */ + { + uint next_length,prev_length,prev_pack_length,lastkey_length, + rest_length; + if (keyinfo->seg[0].length >= 127) + { + if (!(prev_length=mi_uint2korr(start) & 32767)) + goto end; + next_length=mi_uint2korr(keypos) & 32767; + keypos+=2; + prev_pack_length=2; + } + else + { + if (!(prev_length= *start & 127)) + goto end; /* Same key as previous*/ + next_length= *keypos & 127; + keypos++; + prev_pack_length=1; + } + if (!(*start & 128)) + prev_length=0; /* prev key not packed */ + if (keyinfo->seg[0].flag & HA_NULL_PART) + lastkey++; /* Skipp null marker */ + get_key_length(lastkey_length,lastkey); + if (!next_length) /* Same key after */ + { + next_length=lastkey_length; + rest_length=0; + } + else + get_key_length(rest_length,keypos); + + if (next_length > prev_length) + { /* Key after is based on deleted key */ + uint pack_length,tmp; + bmove_upp((char*) keypos,(char*) (lastkey+next_length), + tmp=(next_length-prev_length)); + rest_length+=tmp; + pack_length= prev_length ? get_pack_length(rest_length): 0; + keypos-=tmp+pack_length+prev_pack_length; + s_length=(keypos-start); + if (prev_length) /* Pack against prev key */ + { + *keypos++= start[0]; + if (prev_pack_length == 2) + *keypos++= start[1]; + store_key_length(keypos,rest_length); + } + else + { + /* Next key is not packed anymore */ + if (keyinfo->seg[0].flag & HA_NULL_PART) + { + rest_length++; /* Mark not null */ + } + if (prev_pack_length == 2) + { + mi_int2store(keypos,rest_length); + } + else + *keypos= rest_length; + } + } + } + } + } + } + end: + bmove((byte*) start,(byte*) start+s_length, + (uint) (page_end-start-s_length)); + DBUG_RETURN((uint) s_length); +} /* remove_key */ diff --git a/myisam/mi_delete_all.c b/myisam/mi_delete_all.c new file mode 100644 index 00000000000..c3ed9455e12 --- /dev/null +++ b/myisam/mi_delete_all.c @@ -0,0 +1,65 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Remove all rows from a MyISAM table */ +/* This only clears the status information; The files are not truncated */ + +#include "myisamdef.h" + +int mi_delete_all_rows(MI_INFO *info) +{ + uint i; + MYISAM_SHARE *share=info->s; + MI_STATE_INFO *state=&share->state; + DBUG_ENTER("mi_delete_all_rows"); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + if (_mi_mark_file_changed(info)) + goto err; + + info->state->records=info->state->del=state->split=0; + state->dellink = HA_OFFSET_ERROR; + state->sortkey= (ushort) ~0; + info->state->key_file_length=share->base.keystart; + info->state->data_file_length=0; + info->state->empty=info->state->key_empty=0; + state->checksum=0; + + for (i=share->base.max_key_block_length/MI_KEY_BLOCK_LENGTH ; i-- ; ) + state->key_del[i]= HA_OFFSET_ERROR; + for (i=0 ; i < share->base.keys ; i++) + state->key_root[i]= HA_OFFSET_ERROR; + + myisam_log_command(MI_LOG_DELETE_ALL,info,(byte*) 0,0,0); + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(0); + +err: + { + int save_errno=my_errno; + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + info->update|=HA_STATE_WRITTEN; /* Buffer changed */ + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(my_errno=save_errno); + } +} /* mi_delete */ + diff --git a/myisam/mi_delete_table.c b/myisam/mi_delete_table.c new file mode 100644 index 00000000000..916f5a8fc02 --- /dev/null +++ b/myisam/mi_delete_table.c @@ -0,0 +1,53 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + deletes a table +*/ + +#include "fulltext.h" +#ifdef __WIN__ +#include <errno.h> +#endif + +int mi_delete_table(const char *name) +{ + char from[FN_REFLEN]; +#ifdef USE_RAID + uint raid_type=0,raid_chunks=0; +#endif + DBUG_ENTER("mi_delete_table"); +#ifdef USE_RAID + { + MI_INFO *info; + if (!(info=mi_open(name, O_RDONLY, 0))) + DBUG_RETURN(my_errno); + raid_type = info->s->base.raid_type; + raid_chunks = info->s->base.raid_chunks; + mi_close(info); + } +#endif + + fn_format(from,name,"",MI_NAME_IEXT,4); + if (my_delete(from, MYF(MY_WME))) + DBUG_RETURN(my_errno); + fn_format(from,name,"",MI_NAME_DEXT,4); +#ifdef USE_RAID + if (raid_type) + DBUG_RETURN(my_raid_delete(from, raid_chunks, MYF(MY_WME)) ? my_errno : 0); +#endif + DBUG_RETURN(my_delete(from, MYF(MY_WME)) ? my_errno : 0); +} diff --git a/myisam/mi_dynrec.c b/myisam/mi_dynrec.c new file mode 100644 index 00000000000..789831db810 --- /dev/null +++ b/myisam/mi_dynrec.c @@ -0,0 +1,1537 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + /* Functions to handle space-packed-records and blobs */ + +#include "myisamdef.h" + +/* Enough for comparing if number is zero */ +static char zero_string[]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +static int write_dynamic_record(MI_INFO *info,const byte *record, + ulong reclength); +static int _mi_find_writepos(MI_INFO *info,ulong reclength,my_off_t *filepos, + ulong *length); +static int update_dynamic_record(MI_INFO *info,my_off_t filepos,byte *record, + ulong reclength); +static int delete_dynamic_record(MI_INFO *info,my_off_t filepos, + uint second_read); +static int _mi_cmp_buffer(File file, const byte *buff, my_off_t filepos, + uint length); + +#ifdef THREAD +/* Play it safe; We have a small stack when using threads */ +#undef my_alloca +#undef my_afree +#define my_alloca(A) my_malloc((A),MYF(0)) +#define my_afree(A) my_free((A),MYF(0)) +#endif + + /* Interface function from MI_INFO */ + +int _mi_write_dynamic_record(MI_INFO *info, const byte *record) +{ + ulong reclength=_mi_rec_pack(info,info->rec_buff,record); + return (write_dynamic_record(info,info->rec_buff,reclength)); +} + +int _mi_update_dynamic_record(MI_INFO *info, my_off_t pos, const byte *record) +{ + uint length=_mi_rec_pack(info,info->rec_buff,record); + return (update_dynamic_record(info,pos,info->rec_buff,length)); +} + +int _mi_write_blob_record(MI_INFO *info, const byte *record) +{ + byte *rec_buff; + int error; + ulong reclength,extra; + + extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER+1; + reclength=info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,record)+ extra; + if (reclength > MI_DYN_MAX_ROW_LENGTH) + { + my_errno=HA_ERR_TO_BIG_ROW; + return -1; + } + if (!(rec_buff=(byte*) my_alloca(reclength))) + { + my_errno=ENOMEM; + return(-1); + } + reclength=_mi_rec_pack(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + record); + error=write_dynamic_record(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + reclength); + my_afree(rec_buff); + return(error); +} + + +int _mi_update_blob_record(MI_INFO *info, my_off_t pos, const byte *record) +{ + byte *rec_buff; + int error; + ulong reclength,extra; + + extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER; + reclength=info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,record)+ extra; + if (reclength > MI_DYN_MAX_ROW_LENGTH) + { + my_errno=HA_ERR_TO_BIG_ROW; + return -1; + } + if (!(rec_buff=(byte*) my_alloca(reclength))) + { + my_errno=ENOMEM; + return(-1); + } + reclength=_mi_rec_pack(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + record); + error=update_dynamic_record(info,pos, + rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + reclength); + my_afree(rec_buff); + return(error); +} + + +int _mi_delete_dynamic_record(MI_INFO *info) +{ + return delete_dynamic_record(info,info->lastpos,0); +} + + + /* Write record to data-file */ + +static int write_dynamic_record(MI_INFO *info, const byte *record, + ulong reclength) +{ + int flag; + ulong length; + my_off_t filepos; + DBUG_ENTER("write_dynamic_record"); + + flag=0; + while (reclength) + { + if (_mi_find_writepos(info,reclength,&filepos,&length)) + goto err; + if (_mi_write_part_record(info,filepos,length,info->s->state.dellink, + (byte**) &record,&reclength,&flag)) + goto err; + } + + DBUG_RETURN(0); + err: + DBUG_RETURN(1); +} + + + /* Get a block for data ; The given data-area must be used !! */ + +static int _mi_find_writepos(MI_INFO *info, + ulong reclength, /* record length */ + my_off_t *filepos, /* Return file pos */ + ulong *length) /* length of block at filepos */ +{ + MI_BLOCK_INFO block_info; + DBUG_ENTER("_mi_find_writepos"); + + if (info->s->state.dellink != HA_OFFSET_ERROR) + { + /* Deleted blocks exists; Get last used block */ + + *filepos=info->s->state.dellink; + block_info.second_read=0; + info->rec_cache.seek_not_done=1; + if (!(_mi_get_block_info(&block_info,info->dfile,info->s->state.dellink) & + BLOCK_DELETED)) + { + DBUG_PRINT("error",("Delete link crashed")); + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(-1); + } + info->s->state.dellink=block_info.next_filepos; + info->state->del--; + info->state->empty-= block_info.block_len; + *length= block_info.block_len; + } + else + { + /* No deleted blocks; Allocate a new block */ + *filepos=info->state->data_file_length; + if ((*length=reclength+3 + test(reclength > 65520)) < + info->s->base.min_block_length) + *length=info->s->base.min_block_length; + else + *length= ((*length+MI_DYN_ALIGN_SIZE-1) & + (~ (ulong) (MI_DYN_ALIGN_SIZE-1))); + if (info->state->data_file_length > + (info->s->base.max_data_file_length- *length)) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + DBUG_RETURN(-1); + } + info->state->data_file_length+= *length; + info->s->state.split++; + info->update|=HA_STATE_WRITE_AT_END; + } + DBUG_RETURN(0); +} /* _mi_find_writepos */ + + + +/* Remove a deleted block from the deleted list */ + +static bool unlink_deleted_block(MI_INFO *info, MI_BLOCK_INFO *block_info) +{ + DBUG_ENTER("unlink_deleted_block"); + if (block_info->filepos == info->s->state.dellink) + { + /* First deleted block; We can just use this ! */ + info->s->state.dellink=block_info->next_filepos; + } + else + { + MI_BLOCK_INFO tmp; + tmp.second_read=0; + if (!(_mi_get_block_info(&tmp,info->dfile,block_info->prev_filepos) + & BLOCK_DELETED)) + DBUG_RETURN(1); /* Something is wrong */ + mi_sizestore(tmp.header+4,block_info->next_filepos); + if (my_pwrite(info->dfile,(char*) tmp.header+4,8, + block_info->prev_filepos+4, MYF(MY_NABP))) + DBUG_RETURN(1); + if (block_info->next_filepos != HA_OFFSET_ERROR) + { + if (!(_mi_get_block_info(&tmp,info->dfile,block_info->next_filepos) + & BLOCK_DELETED)) + DBUG_RETURN(1); /* Something is wrong */ + mi_sizestore(tmp.header+12,block_info->prev_filepos); + if (my_pwrite(info->dfile,(char*) tmp.header+12,8, + block_info->next_filepos+12, + MYF(MY_NABP))) + DBUG_RETURN(1); + } + } + info->state->del--; + info->state->empty-= block_info->block_len; + info->s->state.split--; + + /* Removing block that we are using through mi_rrnd */ + if (info->nextpos == block_info->filepos) + info->nextpos+=block_info->block_len; + DBUG_RETURN(0); +} + + /* Delete datarecord from database */ + /* info->rec_cache.seek_not_done is updated in cmp_record */ + +static int delete_dynamic_record(MI_INFO *info, my_off_t filepos, + uint second_read) +{ + uint length,b_type; + MI_BLOCK_INFO block_info,del_block; + int error=0; + my_bool remove_next_block; + DBUG_ENTER("delete_dynamic_record"); + + /* First add a link from the last block to the new one */ + if (info->s->state.dellink != HA_OFFSET_ERROR) + { + block_info.second_read=0; + if (_mi_get_block_info(&block_info,info->dfile,info->s->state.dellink) + & BLOCK_DELETED) + { + char buff[8]; + mi_sizestore(buff,filepos); + if (my_pwrite(info->dfile,buff,8,info->s->state.dellink+12, + MYF(MY_NABP))) + error=1; /* Error on write */ + } + else + { + error=1; /* Wrong delete link */ + my_errno=HA_ERR_WRONG_IN_RECORD; + } + } + + block_info.second_read=second_read; + do + { + /* Remove block at 'filepos' */ + if ((b_type=_mi_get_block_info(&block_info,info->dfile,filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR) || + (length=(uint) (block_info.filepos-filepos) +block_info.block_len) < + MI_MIN_BLOCK_LENGTH) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + /* Check if next block is a delete block */ + del_block.second_read=0; + remove_next_block=0; + if (_mi_get_block_info(&del_block,info->dfile,filepos+length) & + BLOCK_DELETED && del_block.block_len+length < MI_DYN_MAX_BLOCK_LENGTH) + { + /* We can't remove this yet as this block may be the head block */ + remove_next_block=1; + length+=del_block.block_len; + } + + block_info.header[0]=0; + mi_int3store(block_info.header+1,length); + mi_sizestore(block_info.header+4,info->s->state.dellink); + if (b_type & BLOCK_LAST) + bfill(block_info.header+12,8,255); + else + mi_sizestore(block_info.header+12,block_info.next_filepos); + if (my_pwrite(info->dfile,(byte*) block_info.header,20,filepos, + MYF(MY_NABP))) + DBUG_RETURN(1); + info->s->state.dellink = filepos; + info->state->del++; + info->state->empty+=length; + filepos=block_info.next_filepos; + + /* Now it's safe to unlink the block */ + if (remove_next_block && unlink_deleted_block(info,&del_block)) + error=1; + } while (!(b_type & BLOCK_LAST)); + + DBUG_RETURN(error); +} + + + /* Write a block to datafile */ + +int _mi_write_part_record(MI_INFO *info, + my_off_t filepos, /* points at empty block */ + ulong length, /* length of block */ + my_off_t next_filepos,/* Next empty block */ + byte **record, /* pointer to record ptr */ + ulong *reclength, /* length of *record */ + int *flag) /* *flag == 0 if header */ +{ + ulong head_length,res_length,extra_length,long_block,del_length; + byte *pos,*record_end; + my_off_t next_delete_block; + uchar temp[MI_SPLIT_LENGTH+MI_DYN_DELETE_BLOCK_HEADER]; + DBUG_ENTER("_mi_write_part_record"); + + next_delete_block=HA_OFFSET_ERROR; + + res_length=extra_length=0; + if (length > *reclength + MI_SPLIT_LENGTH) + { /* Splitt big block */ + res_length=MY_ALIGN(length- *reclength - MI_EXTEND_BLOCK_LENGTH, + MI_DYN_ALIGN_SIZE); + length-= res_length; /* Use this for first part */ + } + long_block= (length < 65520L && *reclength < 65520L) ? 0 : 1; + if (length == *reclength+ 3 + long_block) + { + /* Block is exactly of the right length */ + temp[0]=(uchar) (1+ *flag)+(uchar) long_block; /* Flag is 0 or 6 */ + if (long_block) + { + mi_int3store(temp+1,*reclength); + head_length=4; + } + else + { + mi_int2store(temp+1,*reclength); + head_length=3; + } + } + else if (length-long_block < *reclength+4) + { /* To short block */ + if (next_filepos == HA_OFFSET_ERROR) + next_filepos=info->s->state.dellink != HA_OFFSET_ERROR ? + info->s->state.dellink : info->state->data_file_length; + if (*flag == 0) /* First block */ + { + head_length=5+8+long_block*2; + temp[0]=5+(uchar) long_block; + if (long_block) + { + mi_int3store(temp+1,*reclength); + mi_int3store(temp+4,length-head_length); + mi_sizestore((byte*) temp+7,next_filepos); + } + else + { + mi_int2store(temp+1,*reclength); + mi_int2store(temp+3,length-head_length); + mi_sizestore((byte*) temp+5,next_filepos); + } + } + else + { + head_length=3+8+long_block; + temp[0]=11+(uchar) long_block; + if (long_block) + { + mi_int3store(temp+1,length-head_length); + mi_sizestore((byte*) temp+4,next_filepos); + } + else + { + mi_int2store(temp+1,length-head_length); + mi_sizestore((byte*) temp+3,next_filepos); + } + } + } + else + { /* Block with empty info last */ + head_length=4+long_block; + extra_length= length- *reclength-head_length; + temp[0]= (uchar) (3+ *flag)+(uchar) long_block; /* 3,4 or 9,10 */ + if (long_block) + { + mi_int3store(temp+1,*reclength); + temp[4]= (uchar) (extra_length); + } + else + { + mi_int2store(temp+1,*reclength); + temp[3]= (uchar) (extra_length); + } + length= *reclength+head_length; /* Write only what is needed */ + } + DBUG_DUMP("header",(byte*) temp,head_length); + + /* Make a long block for one write */ + record_end= *record+length-head_length; + del_length=(res_length ? MI_DYN_DELETE_BLOCK_HEADER : 0); + bmove((byte*) (*record-head_length),(byte*) temp,head_length); + memcpy(temp,record_end,(size_t) (extra_length+del_length)); + bzero((byte*) record_end,extra_length); + + if (res_length) + { + /* Check first if we can join this block with the next one */ + MI_BLOCK_INFO del_block; + my_off_t next_block=filepos+length+extra_length+res_length; + + del_block.second_read=0; + if (next_block < info->state->data_file_length && + info->s->state.dellink != HA_OFFSET_ERROR) + { + if ((_mi_get_block_info(&del_block,info->dfile,next_block) + & BLOCK_DELETED) && + res_length + del_block.block_len < MI_DYN_MAX_BLOCK_LENGTH) + { + if (unlink_deleted_block(info,&del_block)) + goto err; + res_length+=del_block.block_len; + } + } + + /* Create a delete link of the last part of the block */ + pos=record_end+extra_length; + pos[0]= '\0'; + mi_int3store(pos+1,res_length); + mi_sizestore(pos+4,info->s->state.dellink); + bfill(pos+12,8,255); /* End link */ + next_delete_block=info->s->state.dellink; + info->s->state.dellink= filepos+length+extra_length; + info->state->del++; + info->state->empty+=res_length; + info->s->state.split++; + } + if (info->opt_flag & WRITE_CACHE_USED && + info->update & HA_STATE_WRITE_AT_END) + { + if (info->update & HA_STATE_EXTEND_BLOCK) + { + info->update&= ~HA_STATE_EXTEND_BLOCK; + if (my_block_write(&info->rec_cache,(byte*) *record-head_length, + length+extra_length+del_length,filepos)) + goto err; + } + else if (my_b_write(&info->rec_cache,(byte*) *record-head_length, + length+extra_length+del_length)) + goto err; + } + else + { + info->rec_cache.seek_not_done=1; + if (my_pwrite(info->dfile,(byte*) *record-head_length,length+extra_length+ + del_length,filepos,info->s->write_flag)) + goto err; + } + memcpy(record_end,temp,(size_t) (extra_length+del_length)); + *record=record_end; + *reclength-=(length-head_length); + *flag=6; + + if (del_length && next_delete_block != HA_OFFSET_ERROR) + { + /* link the next delete block to this */ + MI_BLOCK_INFO del_block; + del_block.second_read=0; + if (!(_mi_get_block_info(&del_block,info->dfile,next_delete_block) + & BLOCK_DELETED)) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + mi_sizestore(del_block.header+12,info->s->state.dellink); + if (my_pwrite(info->dfile,(char*) del_block.header+12,8, + next_delete_block+12, + MYF(MY_NABP))) + goto err; + } + + DBUG_RETURN(0); +err: + DBUG_PRINT("exit",("errno: %d",my_errno)); + DBUG_RETURN(1); +} /*_mi_write_part_record */ + + + /* update record from datafile */ + +static int update_dynamic_record(MI_INFO *info, my_off_t filepos, byte *record, + ulong reclength) +{ + int flag; + uint error; + ulong length; + MI_BLOCK_INFO block_info; + DBUG_ENTER("update_dynamic_record"); + + flag=block_info.second_read=0; + while (reclength > 0) + { + if (filepos != info->s->state.dellink) + { + block_info.next_filepos= HA_OFFSET_ERROR; + if ((error=_mi_get_block_info(&block_info,info->dfile,filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + DBUG_PRINT("error",("Got wrong block info")); + if (!(error & BLOCK_FATAL_ERROR)) + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + length=(ulong) (block_info.filepos-filepos) + block_info.block_len; + if (length < reclength) + { + uint tmp=MY_ALIGN(reclength - length + 3 + + test(reclength >= 65520L),MI_DYN_ALIGN_SIZE); + /* Check if we can extend this block */ + if (block_info.filepos + block_info.block_len == + info->state->data_file_length && + info->state->data_file_length < + info->s->base.max_data_file_length-tmp) + { + /* extend file */ + DBUG_PRINT("info",("Extending file with %d bytes",tmp)); + if (info->nextpos == info->state->data_file_length) + info->nextpos+= tmp; + info->state->data_file_length+= tmp; + info->update|= HA_STATE_WRITE_AT_END | HA_STATE_EXTEND_BLOCK; + length+=tmp; + } + else + { + /* Check if next block is a deleted block */ + MI_BLOCK_INFO del_block; + del_block.second_read=0; + if (_mi_get_block_info(&del_block,info->dfile, + block_info.filepos + block_info.block_len) & + BLOCK_DELETED) + { + /* Use; Unlink it and extend the current block */ + DBUG_PRINT("info",("Extending current block")); + if (unlink_deleted_block(info,&del_block)) + goto err; + length+=del_block.block_len; + } + } + } + } + else + { + if (_mi_find_writepos(info,reclength,&filepos,&length)) + goto err; + } + if (_mi_write_part_record(info,filepos,length,block_info.next_filepos, + &record,&reclength,&flag)) + goto err; + if ((filepos=block_info.next_filepos) == HA_OFFSET_ERROR) + filepos=info->s->state.dellink; + } + + if (block_info.next_filepos != HA_OFFSET_ERROR) + if (delete_dynamic_record(info,block_info.next_filepos,1)) + goto err; + DBUG_RETURN(0); +err: + DBUG_RETURN(1); +} + + + /* Pack a record. Return new reclength */ + +uint _mi_rec_pack(MI_INFO *info, register byte *to, register const byte *from) +{ + uint length,new_length,flag,bit,i; + char *pos,*end,*startpos,*packpos; + enum en_fieldtype type; + reg3 MI_COLUMNDEF *rec; + MI_BLOB *blob; + DBUG_ENTER("_mi_rec_pack"); + + flag=0 ; bit=1; + startpos=packpos=to; to+= info->s->base.pack_bits; blob=info->blobs; + rec=info->s->rec; + + for (i=info->s->base.fields ; i-- > 0; from+= length,rec++) + { + length=(uint) rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL) + { + if (type == FIELD_BLOB) + { + if (!blob->length) + flag|=bit; + else + { + char *temp_pos; + size_t tmp_length=length-mi_portable_sizeof_char_ptr; + memcpy((byte*) to,from,tmp_length); + memcpy_fixed(&temp_pos,from+tmp_length,sizeof(char*)); + memcpy(to+tmp_length,temp_pos,(size_t) blob->length); + to+=tmp_length+blob->length; + } + blob++; + } + else if (type == FIELD_SKIPP_ZERO) + { + if (memcmp((byte*) from,zero_string,length) == 0) + flag|=bit; + else + { + memcpy((byte*) to,from,(size_t) length); to+=length; + } + } + else if (type == FIELD_SKIPP_ENDSPACE || + type == FIELD_SKIPP_PRESPACE) + { + pos= (byte*) from; end= (byte*) from + length; + if (type == FIELD_SKIPP_ENDSPACE) + { /* Pack trailing spaces */ + while (end > from && *(end-1) == ' ') + end--; + } + else + { /* Pack pref-spaces */ + while (pos < end && *pos == ' ') + pos++; + } + new_length=(uint) (end-pos); + if (new_length +1 + test(rec->length > 255 && new_length > 127) + < length) + { + if (rec->length > 255 && new_length > 127) + { + to[0]=(char) ((new_length & 127)+128); + to[1]=(char) (new_length >> 7); + to+=2; + } + else + *to++= (char) new_length; + memcpy((byte*) to,pos,(size_t) new_length); to+=new_length; + flag|=bit; + } + else + { + memcpy(to,from,(size_t) length); to+=length; + } + } + else if (type == FIELD_VARCHAR) + { + uint tmp_length=uint2korr(from); + store_key_length_inc(to,tmp_length); + memcpy(to,from+2,tmp_length); + to+=tmp_length; + continue; + } + else + { + memcpy(to,from,(size_t) length); to+=length; + continue; /* Normal field */ + } + if ((bit= bit << 1) >= 256) + { + *packpos++ = (char) (uchar) flag; + bit=1; flag=0; + } + } + else + { + memcpy(to,from,(size_t) length); to+=length; + } + } + if (bit != 1) + *packpos= (char) (uchar) flag; + if (info->s->calc_checksum) + *to++=(char) info->checksum; + DBUG_PRINT("exit",("packed length: %d",(int) (to-startpos))); + DBUG_RETURN((uint) (to-startpos)); +} /* _mi_rec_pack */ + + + +/* +** Check if a record was correctly packed. Used only by isamchk +** Returns 0 if record is ok. +*/ + +my_bool _mi_rec_check(MI_INFO *info,const char *record) +{ + uint length,new_length,flag,bit,i; + char *pos,*end,*packpos,*to; + enum en_fieldtype type; + reg3 MI_COLUMNDEF *rec; + DBUG_ENTER("_mi_rec_check"); + + packpos=info->rec_buff; to= info->rec_buff+info->s->base.pack_bits; + rec=info->s->rec; + flag= *packpos; bit=1; + + for (i=info->s->base.fields ; i-- > 0; record+= length, rec++) + { + length=(uint) rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL) + { + if (type == FIELD_BLOB) + { + uint blob_length= + _mi_calc_blob_length(length-mi_portable_sizeof_char_ptr,record); + if (!blob_length && !(flag & bit)) + goto err; + if (blob_length) + to+=length - mi_portable_sizeof_char_ptr+ blob_length; + } + else if (type == FIELD_SKIPP_ZERO) + { + if (memcmp((byte*) record,zero_string,length) == 0) + { + if (!(flag & bit)) + goto err; + } + else + to+=length; + } + else if (type == FIELD_SKIPP_ENDSPACE || + type == FIELD_SKIPP_PRESPACE) + { + pos= (byte*) record; end= (byte*) record + length; + if (type == FIELD_SKIPP_ENDSPACE) + { /* Pack trailing spaces */ + while (end > record && *(end-1) == ' ') + end--; + } + else + { /* Pack pre-spaces */ + while (pos < end && *pos == ' ') + pos++; + } + new_length=(uint) (end-pos); + if (new_length +1 + test(rec->length > 255 && new_length > 127) + < length) + { + if (!(flag & bit)) + goto err; + if (rec->length > 255 && new_length > 127) + { + if (to[0] != (char) ((new_length & 127)+128) || + to[1] != (char) (new_length >> 7)) + goto err; + to+=2; + } + else if (*to++ != (char) new_length) + goto err; + to+=new_length; + } + else + to+=length; + } + else if (type == FIELD_VARCHAR) + { + uint tmp_length=uint2korr(record); + to+=get_pack_length(tmp_length)+tmp_length; + continue; + } + else + { + to+=length; + continue; /* Normal field */ + } + if ((bit= bit << 1) >= 256) + { + flag= *++packpos; + bit=1; + } + } + else + { + to+=length; + } + } + if (info->packed_length != (uint) (to - info->rec_buff) + + test(info->s->calc_checksum) || + (bit != 1 && (flag & ~(bit - 1)))) + goto err; + if (info->s->calc_checksum) + { + if ((uchar) info->checksum != (uchar) *to) + { + DBUG_PRINT("error",("wrong checksum for row")); + goto err; + } + } + DBUG_RETURN(0); + + err: + DBUG_RETURN(1); +} + + + + /* Unpacks a record */ + /* Returns -1 and my_errno =HA_ERR_RECORD_DELETED if reclength isn't */ + /* right. Returns reclength (>0) if ok */ + +ulong _mi_rec_unpack(register MI_INFO *info, register byte *to, byte *from, + ulong found_length) +{ + uint flag,bit,length,rec_length,min_pack_length; + enum en_fieldtype type; + byte *from_end,*to_end,*packpos; + reg3 MI_COLUMNDEF *rec,*end_field; + DBUG_ENTER("_mi_rec_unpack"); + + to_end=to + info->s->base.reclength; + from_end=from+found_length; + flag= (uchar) *from; bit=1; packpos=from; + if (found_length < info->s->base.min_pack_length) + goto err; + from+= info->s->base.pack_bits; + min_pack_length=info->s->base.min_pack_length - info->s->base.pack_bits; + + for (rec=info->s->rec , end_field=rec+info->s->base.fields ; + rec < end_field ; to+= rec_length, rec++) + { + rec_length=rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL && + (type != FIELD_CHECK)) + { + if (type == FIELD_VARCHAR) + { + get_key_length(length,from); + if (length > rec_length-2) + goto err; + int2store(to,length); + memcpy(to+2,from,length); + from+=length; + continue; + } + if (flag & bit) + { + if (type == FIELD_BLOB || type == FIELD_SKIPP_ZERO) + bzero((byte*) to,rec_length); + else if (type == FIELD_SKIPP_ENDSPACE || + type == FIELD_SKIPP_PRESPACE) + { + if (rec->length > 255 && *from & 128) + { + if (from + 1 >= from_end) + goto err; + length= (*from & 127)+ ((uint) (uchar) *(from+1) << 7); from+=2; + } + else + { + if (from == from_end) + goto err; + length= (uchar) *from++; + } + min_pack_length--; + if (length >= rec_length || + min_pack_length + length > (uint) (from_end - from)) + goto err; + if (type == FIELD_SKIPP_ENDSPACE) + { + memcpy(to,(byte*) from,(size_t) length); + bfill((byte*) to+length,rec_length-length,' '); + } + else + { + bfill((byte*) to,rec_length-length,' '); + memcpy(to+rec_length-length,(byte*) from,(size_t) length); + } + from+=length; + } + } + else if (type == FIELD_BLOB) + { + uint size_length=rec_length- mi_portable_sizeof_char_ptr; + ulong blob_length=_mi_calc_blob_length(size_length,from); + if ((ulong) (from_end-from) - size_length < blob_length || + min_pack_length > (uint) (from_end -(from+size_length+blob_length))) + goto err; + memcpy((byte*) to,(byte*) from,(size_t) size_length); + from+=size_length; + memcpy_fixed((byte*) to+size_length,(byte*) &from,sizeof(char*)); + from+=blob_length; + } + else + { + if (type == FIELD_SKIPP_ENDSPACE || type == FIELD_SKIPP_PRESPACE) + min_pack_length--; + if (min_pack_length + rec_length > (uint) (from_end - from)) + goto err; + memcpy(to,(byte*) from,(size_t) rec_length); from+=rec_length; + } + if ((bit= bit << 1) >= 256) + { + flag= (uchar) *++packpos; bit=1; + } + } + else + { + if (min_pack_length > (uint) (from_end - from)) + goto err; + min_pack_length-=rec_length; + memcpy(to,(byte*) from,(size_t) rec_length); from+=rec_length; + } + } + if (info->s->calc_checksum) + from++; + if (to == to_end && from == from_end && (bit == 1 || !(flag & ~(bit-1)))) + DBUG_RETURN((info->packed_length=found_length)); + err: + my_errno=HA_ERR_RECORD_DELETED; + DBUG_PRINT("error",("to_end: %lx -> %lx from_end: %lx -> %lx", + to,to_end,from,from_end)); + DBUG_DUMP("from",(byte*) info->rec_buff,info->s->base.min_pack_length); + DBUG_RETURN(MY_FILE_ERROR); +} /* _mi_rec_unpack */ + + + /* Calc length of blob. Update info in blobs->length */ + +ulong _my_calc_total_blob_length(MI_INFO *info, const byte *record) +{ + ulong length; + MI_BLOB *blob,*end; + + for (length=0, blob= info->blobs, end=blob+info->s->base.blobs ; + blob != end; + blob++) + { + blob->length=_mi_calc_blob_length(blob->pack_length,record + blob->offset); + length+=blob->length; + } + return length; +} + + +ulong _mi_calc_blob_length(uint length, const byte *pos) +{ + switch (length) { + case 1: + return (uint) (uchar) *pos; + case 2: + return (uint) uint2korr(pos); + case 3: + return uint3korr(pos); + case 4: + return uint4korr(pos); + default: + break; + } + return 0; /* Impossible */ +} + + +void _my_store_blob_length(byte *pos,uint pack_length,uint length) +{ + switch (pack_length) { + case 1: + *pos= (uchar) length; + break; + case 2: + int2store(pos,length); + break; + case 3: + int3store(pos,length); + break; + case 4: + int4store(pos,length); + default: + break; + } + return; +} + + + /* Read record from datafile */ + /* Returns 0 if ok, -1 if error */ + +int _mi_read_dynamic_record(MI_INFO *info, my_off_t filepos, byte *buf) +{ + int flag; + uint b_type,left_length; + byte *to; + MI_BLOCK_INFO block_info; + File file; + DBUG_ENTER("mi_read_dynamic_record"); + + if (filepos != HA_OFFSET_ERROR) + { + LINT_INIT(to); + LINT_INIT(left_length); + file=info->dfile; + block_info.next_filepos=filepos; /* for easyer loop */ + flag=block_info.second_read=0; + do + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file <= block_info.next_filepos && + flush_io_cache(&info->rec_cache)) + goto err; + info->rec_cache.seek_not_done=1; + if ((b_type=_mi_get_block_info(&block_info,file, + block_info.next_filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & (BLOCK_SYNC_ERROR | BLOCK_DELETED)) + my_errno=HA_ERR_RECORD_DELETED; + goto err; + } + if (flag == 0) /* First block */ + { + flag=1; + if (block_info.rec_len > (uint) info->s->base.max_pack_length) + goto panic; + if (info->s->base.blobs) + { + if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + goto err; + } + else + to= info->rec_buff; + left_length=block_info.rec_len; + } + if (left_length < block_info.data_len || ! block_info.data_len) + goto panic; /* Wrong linked record */ + if (my_pread(file,(byte*) to,block_info.data_len,block_info.filepos, + MYF(MY_NABP))) + goto panic; + left_length-=block_info.data_len; + to+=block_info.data_len; + } while (left_length); + + info->update|= HA_STATE_AKTIV; /* We have a aktive record */ + VOID(_mi_writeinfo(info,0)); + DBUG_RETURN(_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != + MY_FILE_ERROR ? 0 : -1); + } + VOID(_mi_writeinfo(info,0)); + DBUG_RETURN(-1); /* Wrong data to read */ + +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; +err: + VOID(_mi_writeinfo(info,0)); + DBUG_RETURN(-1); +} + + +byte *mi_fix_rec_buff_for_blob(MI_INFO *info, ulong length) +{ + uint extra; + if (! info->rec_buff || length > info->alloced_rec_buff_length) + { + byte *newptr; + extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER; + if (!(newptr=(byte*) my_realloc((gptr) info->rec_alloc,length+extra, + MYF(MY_ALLOW_ZERO_PTR)))) + return newptr; + info->rec_alloc=newptr; + info->rec_buff=newptr+ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER); + info->alloced_rec_buff_length=length; + } + return info->rec_buff; +} + + + /* compare unique constraint between stored rows */ + +int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const byte *record, my_off_t pos) +{ + byte *rec_buff,*rec_alloc,*old_record; + uint alloced_rec_buff_length; + int error; + DBUG_ENTER("_mi_cmp_dynamic_unique"); + + if (!(old_record=my_alloca(info->s->base.reclength))) + DBUG_RETURN(1); + + /* Don't let the compare destroy blobs that may be in use */ + rec_buff=info->rec_buff; + rec_alloc=info->rec_alloc; + alloced_rec_buff_length=info->alloced_rec_buff_length; + if (info->s->base.blobs) + { + info->rec_buff=0; + info->rec_alloc=0; + info->alloced_rec_buff_length=0; + } + error=_mi_read_dynamic_record(info,pos,old_record); + if (!error) + error=mi_unique_comp(def, record, old_record, def->null_are_equal); + if (info->s->base.blobs) + { + my_free(info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); + info->rec_buff=rec_buff; + info->rec_alloc=rec_alloc; + info->alloced_rec_buff_length=alloced_rec_buff_length; + } + my_afree(old_record); + DBUG_RETURN(error); +} + + + /* Compare of record one disk with packed record in memory */ + +int _mi_cmp_dynamic_record(register MI_INFO *info, register const byte *record) +{ + uint flag,reclength,b_type; + my_off_t filepos; + byte *buffer; + MI_BLOCK_INFO block_info; + DBUG_ENTER("_mi_cmp_dynamic_record"); + + /* We are going to do changes; dont let anybody disturb */ + dont_break(); /* Dont allow SIGHUP or SIGINT */ + + if (info->opt_flag & WRITE_CACHE_USED) + { + info->update&= ~(HA_STATE_WRITE_AT_END | HA_STATE_EXTEND_BLOCK); + if (flush_io_cache(&info->rec_cache)) + DBUG_RETURN(-1); + } + info->rec_cache.seek_not_done=1; + + /* If nobody have touched the database we don't have to test rec */ + + buffer=info->rec_buff; + if ((info->opt_flag & READ_CHECK_USED)) + { /* If check isn't disabled */ + if (info->s->base.blobs) + { + if (!(buffer=(byte*) my_alloca(info->s->base.pack_reclength+ + _my_calc_total_blob_length(info,record)))) + DBUG_RETURN(-1); + } + reclength=_mi_rec_pack(info,buffer,record); + record= buffer; + + filepos=info->lastpos; + flag=block_info.second_read=0; + block_info.next_filepos=filepos; + while (reclength > 0) + { + if ((b_type=_mi_get_block_info(&block_info,info->dfile, + block_info.next_filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & (BLOCK_SYNC_ERROR | BLOCK_DELETED)) + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + if (flag == 0) /* First block */ + { + flag=1; + if (reclength != block_info.rec_len) + { + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + } else if (reclength < block_info.data_len) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + reclength-=block_info.data_len; + if (_mi_cmp_buffer(info->dfile,record,block_info.filepos, + block_info.data_len)) + { + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + flag=1; + record+=block_info.data_len; + } + } + my_errno=0; + err: + if (buffer != info->rec_buff) + my_afree((gptr) buffer); + DBUG_RETURN(my_errno); +} + + + /* Compare file to buffert */ + +static int _mi_cmp_buffer(File file, const byte *buff, my_off_t filepos, + uint length) +{ + uint next_length; + char temp_buff[IO_SIZE*2]; + DBUG_ENTER("_mi_cmp_buffer"); + + VOID(my_seek(file,filepos,MY_SEEK_SET,MYF(0))); + next_length= IO_SIZE*2 - (uint) (filepos & (IO_SIZE-1)); + + while (length > IO_SIZE*2) + { + if (my_read(file,temp_buff,next_length,MYF(MY_NABP))) + goto err; + if (memcmp((byte*) buff,temp_buff,IO_SIZE)) + DBUG_RETURN(1); + buff+=next_length; + length-= next_length; + next_length=IO_SIZE*2; + } + if (my_read(file,temp_buff,length,MYF(MY_NABP))) + goto err; + DBUG_RETURN(memcmp((byte*) buff,temp_buff,length)); +err: + DBUG_RETURN(1); +} + + +int _mi_read_rnd_dynamic_record(MI_INFO *info, byte *buf, + register my_off_t filepos, + my_bool skipp_deleted_blocks) +{ + int flag,info_read,save_errno; + uint left_len,b_type; + byte *to; + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_dynamic_record"); + + info_read=0; + LINT_INIT(to); + + if (info->lock_type == F_UNLCK) + { +#ifndef UNSAFE_LOCKING + if (share->r_locks == 0 && share->w_locks == 0) + { + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) + DBUG_RETURN(my_errno); + } +#else + info->tmp_lock_type=F_RDLCK; +#endif + } + else + info_read=1; /* memory-keyinfoblock is ok */ + + flag=block_info.second_read=0; + left_len=1; + do + { + if (filepos >= info->state->data_file_length) + { + if (!info_read) + { /* Check if changed */ + info_read=1; + info->rec_cache.seek_not_done=1; + if (mi_state_info_read_dsk(share->kfile,&share->state,1)) + goto panic; + } + if (filepos >= info->state->data_file_length) + { + my_errno= HA_ERR_END_OF_FILE; + goto err; + } + } + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(byte*) block_info.header,filepos, + sizeof(block_info.header), + test(!flag && skipp_deleted_blocks) | 2)) + goto panic; + b_type=_mi_get_block_info(&block_info,-1,filepos); + } + else + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file <= filepos && + flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + info->rec_cache.seek_not_done=1; + b_type=_mi_get_block_info(&block_info,info->dfile,filepos); + } + + if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if ((b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) + && skipp_deleted_blocks) + { + filepos=block_info.filepos+block_info.block_len; + block_info.second_read=0; + continue; /* Search after next_record */ + } + if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) + { + my_errno=HA_ERR_RECORD_DELETED; + info->lastpos=block_info.filepos; + info->nextpos=block_info.filepos+block_info.block_len; + } + goto err; + } + if (flag == 0) /* First block */ + { + if (block_info.rec_len > (uint) share->base.max_pack_length) + goto panic; + info->lastpos=filepos; + if (share->base.blobs) + { + if (!(to=mi_fix_rec_buff_for_blob(info,block_info.rec_len))) + goto err; + } + else + to= info->rec_buff; + left_len=block_info.rec_len; + } + if (left_len < block_info.data_len) + goto panic; /* Wrong linked record */ + + /* copy information that is already read */ + { + uint offset=(uint) (block_info.filepos - filepos); + uint tmp_length= (sizeof(block_info.header) - offset); + filepos=block_info.filepos; + + if (tmp_length > block_info.data_len) + tmp_length= block_info.data_len; + if (tmp_length) + { + memcpy((byte*) to, block_info.header+offset,tmp_length); + block_info.data_len-=tmp_length; + left_len-=tmp_length; + to+=tmp_length; + filepos+=tmp_length; + } + } + /* read rest of record from file */ + if (block_info.data_len) + { + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(byte*) to,filepos, + block_info.data_len, + test(!flag && skipp_deleted_blocks))) + goto panic; + } + else + { + /* VOID(my_seek(info->dfile,filepos,MY_SEEK_SET,MYF(0))); */ + if (my_read(info->dfile,(byte*) to,block_info.data_len,MYF(MY_NABP))) + { + if (my_errno == -1) + my_errno= HA_ERR_WRONG_IN_RECORD; /* Unexpected end of file */ + goto err; + } + } + } + if (flag++ == 0) + { + info->nextpos=block_info.filepos+block_info.block_len; + skipp_deleted_blocks=0; + } + left_len-=block_info.data_len; + to+=block_info.data_len; + filepos=block_info.next_filepos; + } while (left_len); + + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + if (share->r_locks == 0 && share->w_locks == 0) + VOID(_mi_writeinfo(info,0)); + if (_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != + MY_FILE_ERROR) + DBUG_RETURN(0); + DBUG_RETURN(my_errno); /* Wrong record */ + +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; /* Something is fatal wrong */ +err: + save_errno=my_errno; + VOID(_mi_writeinfo(info,0)); + DBUG_RETURN(my_errno=save_errno); +} + + + /* Read and process header from a dynamic-record-file */ + +uint _mi_get_block_info(MI_BLOCK_INFO *info, File file, my_off_t filepos) +{ + uint return_val=0; + uchar *header=info->header; + + if (file >= 0) + { + VOID(my_seek(file,filepos,MY_SEEK_SET,MYF(0))); + if (my_read(file,(char*) header,sizeof(info->header),MYF(0)) != + sizeof(info->header)) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + return BLOCK_FATAL_ERROR; + } + } + DBUG_DUMP("header",(byte*) header,MI_BLOCK_INFO_HEADER_LENGTH); + if (info->second_read) + { + if (info->header[0] <= 6) + return_val=BLOCK_SYNC_ERROR; + } + else + { + if (info->header[0] > 6) + return_val=BLOCK_SYNC_ERROR; + } + info->next_filepos= HA_OFFSET_ERROR; /* Dummy ifall no next block */ + + switch (info->header[0]) { + case 0: + if ((info->block_len=(uint) mi_uint3korr(header+1)) < + MI_MIN_BLOCK_LENGTH || + (info->block_len & (MI_DYN_ALIGN_SIZE -1))) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + return BLOCK_ERROR; + } + info->filepos=filepos; + info->next_filepos=mi_sizekorr(header+4); + info->prev_filepos=mi_sizekorr(header+12); +#if SIZEOF_OFF_T == 4 + if ((mi_uint4korr(header+4) != 0 && + (mi_uint4korr(header+4) != (ulong) ~0 || + info->next_filepos != (ulong) ~0)) || + (mi_uint4korr(header+12) != 0 && + (mi_uint4korr(header+12) != (ulong) ~0 || + info->prev_filepos != (ulong) ~0))) + return BLOCK_FATAL_ERROR; +#endif + return return_val | BLOCK_DELETED; /* Deleted block */ + + case 1: + info->rec_len=info->data_len=info->block_len=mi_uint2korr(header+1); + info->filepos=filepos+3; + return return_val | BLOCK_FIRST | BLOCK_LAST; + case 2: + info->rec_len=info->data_len=info->block_len=mi_uint3korr(header+1); + info->filepos=filepos+4; + return return_val | BLOCK_FIRST | BLOCK_LAST; + + case 3: + info->rec_len=info->data_len=mi_uint2korr(header+1); + info->block_len=info->rec_len+ (uint) header[3]; + info->filepos=filepos+4; + return return_val | BLOCK_FIRST | BLOCK_LAST; + case 4: + info->rec_len=info->data_len=mi_uint3korr(header+1); + info->block_len=info->rec_len+ (uint) header[4]; + info->filepos=filepos+5; + return return_val | BLOCK_FIRST | BLOCK_LAST; + + case 5: + info->rec_len=mi_uint2korr(header+1); + info->block_len=info->data_len=mi_uint2korr(header+3); + info->next_filepos=mi_sizekorr(header+5); + info->second_read=1; + info->filepos=filepos+13; + return return_val | BLOCK_FIRST; + case 6: + info->rec_len=mi_uint3korr(header+1); + info->block_len=info->data_len=mi_uint3korr(header+4); + info->next_filepos=mi_sizekorr(header+7); + info->second_read=1; + info->filepos=filepos+15; + return return_val | BLOCK_FIRST; + + /* The following blocks are identical to 1-6 without rec_len */ + case 7: + info->data_len=info->block_len=mi_uint2korr(header+1); + info->filepos=filepos+3; + return return_val | BLOCK_LAST; + case 8: + info->data_len=info->block_len=mi_uint3korr(header+1); + info->filepos=filepos+4; + return return_val | BLOCK_LAST; + + case 9: + info->data_len=mi_uint2korr(header+1); + info->block_len=info->data_len+ (uint) header[3]; + info->filepos=filepos+4; + return return_val | BLOCK_LAST; + case 10: + info->data_len=mi_uint3korr(header+1); + info->block_len=info->data_len+ (uint) header[4]; + info->filepos=filepos+5; + return return_val | BLOCK_LAST; + + case 11: + info->data_len=info->block_len=mi_uint2korr(header+1); + info->next_filepos=mi_sizekorr(header+3); + info->second_read=1; + info->filepos=filepos+11; + return return_val; + case 12: + info->data_len=info->block_len=mi_uint3korr(header+1); + info->next_filepos=mi_sizekorr(header+4); + info->second_read=1; + info->filepos=filepos+12; + return return_val; + default: + my_errno=HA_ERR_WRONG_IN_RECORD; /* Garbage */ + return BLOCK_ERROR; + } +} diff --git a/myisam/mi_extra.c b/myisam/mi_extra.c new file mode 100644 index 00000000000..287f1dd34c8 --- /dev/null +++ b/myisam/mi_extra.c @@ -0,0 +1,324 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Extra functions we want to do with a database */ +/* - Set flags for quicker databasehandler */ +/* - Set databasehandler to normal */ +/* - Reset recordpointers as after open database */ + +#include "myisamdef.h" +#ifdef HAVE_MMAP +#include <sys/mman.h> +#endif +#ifdef __WIN__ +#include <errno.h> +#endif + + /* set extra flags for database */ + +int mi_extra(MI_INFO *info, enum ha_extra_function function) +{ + int error=0; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_extra"); + + switch (function) { + case HA_EXTRA_RESET: + info->lastinx= 0; /* Use first index as def */ + info->last_search_keypage=info->lastpos= HA_OFFSET_ERROR; + info->page_changed=1; + /* Next/prev gives first/last */ + if (info->opt_flag & READ_CACHE_USED) + { + reinit_io_cache(&info->rec_cache,READ_CACHE,0, + (pbool) (info->lock_type != F_UNLCK), + (pbool) test(info->update & HA_STATE_ROW_CHANGED) + ); + } + info->update= ((info->update & HA_STATE_CHANGED) | HA_STATE_NEXT_FOUND | + HA_STATE_PREV_FOUND); + break; + case HA_EXTRA_CACHE: + if (info->lock_type == F_UNLCK && + (share->options & HA_OPTION_PACK_RECORD)) + { + error=1; /* Not possibly if not locked */ + my_errno=EACCES; + break; + } +#if defined(HAVE_MMAP) && defined(HAVE_MADVICE) + if ((share->options & HA_OPTION_COMPRESS_RECORD)) + { + pthread_mutex_lock(&share->intern_lock); + if (_mi_memmap_file(info)) + { + /* We don't nead MADV_SEQUENTIAL if small file */ + madvise(share->file_map,share->state.state.data_file_length, + share->state.state.data_file_length <= RECORD_CACHE_SIZE*16 ? + MADV_RANDOM : MADV_SEQUENTIAL); + pthread_mutex_unlock(&share->intern_lock); + break; + } + pthread_mutex_unlock(&share->intern_lock); + } +#endif + if (info->opt_flag & WRITE_CACHE_USED) + { + info->opt_flag&= ~WRITE_CACHE_USED; + if ((error=end_io_cache(&info->rec_cache))) + break; + } + if (!(info->opt_flag & + (READ_CACHE_USED | WRITE_CACHE_USED | MEMMAP_USED))) + { + if (!(init_io_cache(&info->rec_cache,info->dfile, + (uint) min(info->state->data_file_length+1, + my_default_record_cache_size), + READ_CACHE,0L,(pbool) (info->lock_type != F_UNLCK), + MYF(share->write_flag & MY_WAIT_IF_FULL)))) + { + info->opt_flag|=READ_CACHE_USED; + info->update&= ~HA_STATE_ROW_CHANGED; + } + if (share->concurrent_insert) + info->rec_cache.end_of_file=info->state->data_file_length; + } + break; + case HA_EXTRA_REINIT_CACHE: + if (info->opt_flag & READ_CACHE_USED) + { + reinit_io_cache(&info->rec_cache,READ_CACHE,info->nextpos, + (pbool) (info->lock_type != F_UNLCK), + (pbool) test(info->update & HA_STATE_ROW_CHANGED)); + info->update&= ~HA_STATE_ROW_CHANGED; + if (share->concurrent_insert) + info->rec_cache.end_of_file=info->state->data_file_length; + } + break; + case HA_EXTRA_WRITE_CACHE: + if (info->lock_type == F_UNLCK) + { + error=1; /* Not possibly if not locked */ + break; + } + if (!(info->opt_flag & + (READ_CACHE_USED | WRITE_CACHE_USED | OPT_NO_ROWS)) && + !share->state.header.uniques) + if (!(init_io_cache(&info->rec_cache,info->dfile,0, + WRITE_CACHE,info->state->data_file_length, + (pbool) (info->lock_type != F_UNLCK), + MYF(share->write_flag & MY_WAIT_IF_FULL)))) + { + info->opt_flag|=WRITE_CACHE_USED; + info->update&= ~(HA_STATE_ROW_CHANGED | + HA_STATE_WRITE_AT_END | + HA_STATE_EXTEND_BLOCK); + } + break; + case HA_EXTRA_NO_CACHE: + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + error=end_io_cache(&info->rec_cache); + } +#if defined(HAVE_MMAP) && defined(HAVE_MADVICE) + if (info->opt_flag & MEMMAP_USED) + madvise(share->file_map,share->state.state.data_file_length,MADV_RANDOM); +#endif + break; + case HA_EXTRA_FLUSH_CACHE: + if (info->opt_flag & WRITE_CACHE_USED) + { + if ((error=flush_io_cache(&info->rec_cache))) + mi_mark_crashed(info); /* Fatal error found */ + } + break; + case HA_EXTRA_NO_READCHECK: + info->opt_flag&= ~READ_CHECK_USED; /* No readcheck */ + break; + case HA_EXTRA_READCHECK: + info->opt_flag|= READ_CHECK_USED; + break; + case HA_EXTRA_KEYREAD: /* Read only keys to record */ + case HA_EXTRA_REMEMBER_POS: + info->opt_flag |= REMEMBER_OLD_POS; + bmove((byte*) info->lastkey+share->base.max_key_length*2, + (byte*) info->lastkey,info->lastkey_length); + info->save_update= info->update; + info->save_lastinx= info->lastinx; + info->save_lastpos= info->lastpos; + info->save_lastkey_length=info->lastkey_length; + if (function == HA_EXTRA_REMEMBER_POS) + break; + /* fall through */ + case HA_EXTRA_KEYREAD_CHANGE_POS: + info->opt_flag |= KEY_READ_USED; + info->read_record=_mi_read_key_record; + break; + case HA_EXTRA_NO_KEYREAD: + case HA_EXTRA_RESTORE_POS: + if (info->opt_flag & REMEMBER_OLD_POS) + { + bmove((byte*) info->lastkey, + (byte*) info->lastkey+share->base.max_key_length*2, + info->save_lastkey_length); + info->update= info->save_update | HA_STATE_WRITTEN; + info->lastinx= info->save_lastinx; + info->lastpos= info->save_lastpos; + info->lastkey_length=info->save_lastkey_length; + } + info->read_record= share->read_record; + info->opt_flag&= ~(KEY_READ_USED | REMEMBER_OLD_POS); + break; + case HA_EXTRA_NO_USER_CHANGE: /* Database is somehow locked agains changes */ + info->lock_type= F_EXTRA_LCK; /* Simulate as locked */ + break; + case HA_EXTRA_WAIT_LOCK: + info->lock_wait=0; + break; + case HA_EXTRA_NO_WAIT_LOCK: + info->lock_wait=MY_DONT_WAIT; + break; + case HA_EXTRA_NO_KEYS: + if (info->lock_type == F_UNLCK) + { + error=1; /* Not possibly if not lock */ + break; + } + if (share->state.key_map) + { + share->state.key_map=0; + info->state->key_file_length=share->state.state.key_file_length= + share->base.keystart; + if (!share->changed) + { + share->state.changed|=1; + share->changed=1; /* Update on close */ + if (!share->global_changed) + { + share->global_changed=1; + share->state.open_count++; + } + } + share->state.state= *info->state; + error=mi_state_info_write(share->kfile,&share->state,1 | 2); + } + break; + case HA_EXTRA_FORCE_REOPEN: + pthread_mutex_lock(&THR_LOCK_myisam); + share->last_version= 0L; /* Impossible version */ +#ifdef __WIN__ + /* Close the isam and data files as Win32 can't drop an open table */ + pthread_mutex_lock(&share->intern_lock); + if (flush_key_blocks(share->kfile,FLUSH_RELEASE)) + { + error=my_errno; + share->changed=1; + mi_mark_crashed(info); /* Fatal error found */ + } + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + error=end_io_cache(&info->rec_cache); + } + if (info->lock_type != F_UNLCK && ! info->was_locked) + { + info->was_locked=info->lock_type; + if (mi_lock_database(info,F_UNLCK)) + error=my_errno; + info->lock_type = F_UNLCK; + } + if (share->kfile >= 0 && my_close(share->kfile,MYF(0))) + error=my_errno; + { + LIST *list_element ; + for (list_element=myisam_open_list ; + list_element ; + list_element=list_element->next) + { + MI_INFO *tmpinfo=(MI_INFO*) list_element->data; + if (tmpinfo->s == info->s) + { + if (tmpinfo->dfile >= 0 && my_close(tmpinfo->dfile,MYF(0))) + error = my_errno; + tmpinfo->dfile= -1; + } + } + } + share->kfile= -1; /* Files aren't open anymore */ + pthread_mutex_unlock(&share->intern_lock); +#endif + pthread_mutex_unlock(&THR_LOCK_myisam); + break; + case HA_EXTRA_FLUSH: + if (!share->temporary) + flush_key_blocks(share->kfile,FLUSH_KEEP); +#ifdef HAVE_PWRITE + _mi_decrement_open_count(info); +#endif + if (share->not_flushed) + { + share->not_flushed=0; +#if defined(__WIN__) + if (_commit(share->kfile)) + error=errno; + if (_commit(info->dfile)) + error=errno; +#elif defined(HAVE_FDATASYNC) + if (fdatasync(share->kfile)) + error=errno; + if (fdatasync(share->dfile)) + error=errno; +#elif defined(HAVE_FSYNC) + if ( fsync(share->kfile)) + error=errno; + if (fsync(share->dfile)) + error=errno; +#endif + if (error) + { + share->changed=1; + mi_mark_crashed(info); /* Fatal error found */ + } + } + if (share->base.blobs) + { + my_free(info->rec_alloc,MYF(MY_ALLOW_ZERO_PTR)); + info->rec_alloc=info->rec_buff=0; + } + break; + case HA_EXTRA_NORMAL: /* Theese isn't in use */ + info->quick_mode=0; + break; + case HA_EXTRA_QUICK: + info->quick_mode=1; + break; + case HA_EXTRA_NO_ROWS: + if (!share->state.header.uniques) + info->opt_flag|= OPT_NO_ROWS; + break; + case HA_EXTRA_KEY_CACHE: + case HA_EXTRA_NO_KEY_CACHE: + default: + break; + } + { + char tmp[1]; + tmp[0]=function; + myisam_log_command(MI_LOG_EXTRA,info,(byte*) tmp,1,error); + } + DBUG_RETURN(error); +} /* mi_extra */ diff --git a/myisam/mi_info.c b/myisam/mi_info.c new file mode 100644 index 00000000000..958c881158d --- /dev/null +++ b/myisam/mi_info.c @@ -0,0 +1,102 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Ger tillbaka en struct med information om isam-filen */ + +#include "myisamdef.h" +#ifdef __WIN__ +#include <sys/stat.h> +#endif + + /* Get position to last record */ + +my_off_t mi_position(MI_INFO *info) +{ + return info->lastpos; +} + + +/* Get information about the table */ +/* if flag == 2 one get current info (no sync from database */ + +int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag) +{ + MY_STAT state; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_status"); + + x->recpos = info->lastpos; + if (flag == HA_STATUS_POS) + DBUG_RETURN(0); /* Compatible with ISAM */ + if (!(flag & HA_STATUS_NO_LOCK)) + { + pthread_mutex_lock(&share->intern_lock); + VOID(_mi_readinfo(info,F_RDLCK,0)); + VOID(_mi_writeinfo(info,0)); + pthread_mutex_unlock(&share->intern_lock); + } + if (flag & HA_STATUS_VARIABLE) + { + x->records = info->state->records; + x->deleted = info->state->del; + x->delete_length = info->state->empty; + x->data_file_length =info->state->data_file_length; + x->index_file_length=info->state->key_file_length; + + x->keys = share->state.header.keys; + x->key_map = share->state.key_map; + x->check_time = share->state.check_time; + x->mean_reclength = info->state->records ? + (ulong) (info->state->data_file_length-info->state->empty)/ + info->state->records : (ulong) share->min_pack_length; + } + if (flag & HA_STATUS_ERRKEY) + { + x->errkey = info->errkey; + x->dupp_key_pos= info->dupp_key_pos; + } + if (flag & HA_STATUS_CONST) + { + x->reclength = share->base.reclength; + x->max_data_file_length=share->base.max_data_file_length; + x->max_index_file_length=info->s->base.max_key_file_length; + x->filenr = info->dfile; + x->options = share->options; + x->create_time=share->state.create_time; + x->reflength= mi_get_pointer_length(share->base.max_data_file_length,4); + x->record_offset= ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? + 0L : share->base.pack_reclength); + x->sortkey= -1; /* No clustering */ + /* The following should be included even if we are not compiling with + USE_RAID as the client must be able to request it! */ + x->rec_per_key = share->state.rec_per_key_part; + x->raid_type= share->base.raid_type; + x->raid_chunks= share->base.raid_chunks; + x->raid_chunksize= share->base.raid_chunksize; + } + if ((flag & HA_STATUS_TIME) && !my_fstat(info->dfile,&state,MYF(0))) + x->update_time=state.st_mtime; + else + x->update_time=0; + if (flag & HA_STATUS_AUTO) + { + x->auto_increment= share->state.auto_increment+1; + if (!x->auto_increment) /* This shouldn't happen */ + x->auto_increment= ~(ulonglong) 0; + } + DBUG_RETURN(0); +} diff --git a/myisam/mi_key.c b/myisam/mi_key.c new file mode 100644 index 00000000000..f9eb9b77835 --- /dev/null +++ b/myisam/mi_key.c @@ -0,0 +1,411 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Functions to handle keys */ + +#include "myisamdef.h" +#include "m_ctype.h" + +#define CHECK_KEYS + +static int _mi_put_key_in_record(MI_INFO *info,uint keynr,byte *record); + + /* + ** Make a intern key from a record + ** Ret: Length of key + */ + +uint _mi_make_key(register MI_INFO *info, uint keynr, uchar *key, + const byte *record, my_off_t filepos) +{ + byte *pos,*end; + uchar *start; + reg1 MI_KEYSEG *keyseg; + DBUG_ENTER("_mi_make_key"); + + start=key; + for (keyseg=info->s->keyinfo[keynr].seg ; keyseg->type ;keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + uint length=keyseg->length; + + if (keyseg->null_bit) + { + if (record[keyseg->null_pos] & keyseg->null_bit) + { + *key++= 0; /* NULL in key */ + continue; + } + *key++=1; /* Not NULL */ + } + + pos= (byte*) record+keyseg->start; + if (keyseg->flag & HA_SPACE_PACK) + { + end=pos+length; + if (type != HA_KEYTYPE_NUM) + { + while (end > pos && end[-1] == ' ') + end--; + } + else + { + while (pos < end && pos[0] == ' ') + pos++; + } + length=(uint) (end-pos); + store_key_length_inc(key,length); + memcpy((byte*) key,(byte*) pos,(size_t) length); + key+=length; + continue; + } + if (keyseg->flag & HA_VAR_LENGTH) + { + uint tmp_length=uint2korr(pos); + pos+=2; /* Skip VARCHAR length */ + set_if_smaller(length,tmp_length); + store_key_length_inc(key,length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint tmp_length=_mi_calc_blob_length(keyseg->bit_start,pos); + memcpy_fixed((byte*) &pos,pos+keyseg->bit_start,sizeof(char*)); + set_if_smaller(length,tmp_length); + store_key_length_inc(key,length); + } + else if (keyseg->flag & HA_SWAP_KEY) + { /* Numerical column */ +#ifdef NAN_TEST + float float_nr; + double dbl_nr; + if (type == HA_KEYTYPE_FLOAT) + { + float_nr=float4get(pos); + if (float_nr == (float) FLT_MAX) + { + float_nr= (float) FLT_MAX; + pos= (byte*) &float_nr; + } + } + else if (type == HA_KEYTYPE_DOUBLE) + { + dbl_nr=float8get(key); + if (dbl_nr == DBL_MAX) + { + dbl_nr=DBL_MAX; + pos=(byte*) &dbl_nr; + } + } +#endif + pos+=length; + while (length--) + { + *key++ = *--pos; + } + continue; + } + memcpy((byte*) key, pos, length); + key+= length; + } + _mi_dpointer(info,key,filepos); + DBUG_PRINT("exit",("keynr: %d",keynr)); + DBUG_DUMP("key",(byte*) start,(uint) (key-start)+keyseg->length); + DBUG_EXECUTE("key", + _mi_print_key(DBUG_FILE,info->s->keyinfo[keynr].seg,start, + (uint) (key-start));); + DBUG_RETURN((uint) (key-start)); /* Return keylength */ +} /* _mi_make_key */ + + + /* Pack a key to intern format from given format (c_rkey) */ + /* returns length of packed key */ + +uint _mi_pack_key(register MI_INFO *info, uint keynr, uchar *key, uchar *old, + uint k_length) +{ + uint length; + uchar *pos,*end,*start_key=key; + reg1 MI_KEYSEG *keyseg; + enum ha_base_keytype type; + DBUG_ENTER("_mi_pack_key"); + + start_key=key; + for (keyseg=info->s->keyinfo[keynr].seg ; + keyseg->type && (int) k_length > 0; + old+=keyseg->length, keyseg++) + { + length=min((uint) keyseg->length,(uint) k_length); + type=(enum ha_base_keytype) keyseg->type; + if (keyseg->null_bit) + { + k_length--; + if (!(*key++= (char) 1-*old++)) /* Copy null marker */ + { + k_length-=length; + continue; /* Found NULL */ + } + } + pos=old; + if (keyseg->flag & HA_SPACE_PACK) + { + end=pos+length; + if (type != HA_KEYTYPE_NUM) + { + while (end > pos && end[-1] == ' ') + end--; + } + else + { + while (pos < end && pos[0] == ' ') + pos++; + } + k_length-=length; + length=(uint) (end-pos); + store_key_length_inc(key,length); + memcpy((byte*) key,pos,(size_t) length); + key+= length; + continue; + } + else if (keyseg->flag & (HA_VAR_LENGTH | HA_BLOB_PART)) + { + uint tmp_length=uint2korr(pos); pos+=2; + set_if_smaller(length,tmp_length); + store_key_length_inc(key,length); + k_length-=2; + } + else if (keyseg->flag & HA_SWAP_KEY) + { /* Numerical column */ + pos+=length; + k_length-=length; + while (length--) + { + *key++ = *--pos; + } + continue; + } + memcpy((byte*) key,pos,(size_t) length); + key+= length; + k_length-=length; + } + +#ifdef NOT_USED + if (keyseg->type) + { + /* Part-key ; fill with ASCII 0 for easier searching */ + length= (uint) -k_length; /* unused part of last key */ + do + { + if (keyseg->flag & HA_NULL_PART) + length++; + if (keyseg->flag & HA_SPACE_PACK) + length+=2; + else + length+= keyseg->length; + keyseg++; + } while (keyseg->type); + bzero((byte*) key,length); + key+=length; + } +#endif + DBUG_RETURN((uint) (key-start_key)); +} /* _mi_pack_key */ + + + /* Put a key in record */ + /* Used when only-keyread is wanted */ + +static int _mi_put_key_in_record(register MI_INFO *info, uint keynr, + byte *record) +{ + reg2 byte *key; + byte *pos,*key_end; + reg1 MI_KEYSEG *keyseg; + byte *blob_ptr; + DBUG_ENTER("_mi_put_key_in_record"); + + if (info->blobs && info->s->keyinfo[keynr].flag & HA_VAR_LENGTH_KEY) + { + if (!(blob_ptr= + mi_fix_rec_buff_for_blob(info, info->s->keyinfo[keynr].keylength))) + goto err; + } + key=(byte*) info->lastkey; + key_end=key+info->lastkey_length; + for (keyseg=info->s->keyinfo[keynr].seg ; keyseg->type ;keyseg++) + { + if (keyseg->null_bit) + { + if (!*key++) + { + record[keyseg->null_pos]|= keyseg->null_bit; + continue; + } + record[keyseg->null_pos]&= ~keyseg->null_bit; + } + if (keyseg->flag & HA_SPACE_PACK) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length > keyseg->length || key+length > key_end) + goto err; +#endif + pos= record+keyseg->start; + if (keyseg->type != (int) HA_KEYTYPE_NUM) + { + memcpy(pos,key,(size_t) length); + bfill(pos+length,keyseg->length-length,' '); + } + else + { + bfill(pos,keyseg->length-length,' '); + memcpy(pos+keyseg->length-length,key,(size_t) length); + } + key+=length; + continue; + } + + if (keyseg->flag & HA_VAR_LENGTH) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length >= keyseg->length || key+length > key_end) + goto err; +#endif + memcpy(record+keyseg->start,(byte*) key, length); + key+= length; + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length >= keyseg->length || key+length > key_end) + goto err; +#endif + memcpy(record+keyseg->start+keyseg->bit_start, + (char*) &blob_ptr,sizeof(char*)); + memcpy(blob_ptr,key,length); + blob_ptr+=length; + _my_store_blob_length(record+keyseg->start, + (uint) keyseg->bit_start,length); + key+=length; + } + else if (keyseg->flag & HA_SWAP_KEY) + { + byte *to= record+keyseg->start+keyseg->length; + byte *end= key+keyseg->length; +#ifdef CHECK_KEYS + if (end > key_end) + goto err; +#endif + do + { + *--to= *key++; + } while (key != end); + continue; + } + else + { +#ifdef CHECK_KEYS + if (key+keyseg->length > key_end) + goto err; +#endif + memcpy(record+keyseg->start,(byte*) key, + (size_t) keyseg->length); + key+= keyseg->length; + } + } + DBUG_RETURN(0); + +err: + DBUG_RETURN(1); /* Crashed row */ +} /* _mi_put_key_in_record */ + + + /* Here when key reads are used */ + +int _mi_read_key_record(MI_INFO *info, my_off_t filepos, byte *buf) +{ + VOID(_mi_writeinfo(info,0)); + if (filepos != HA_OFFSET_ERROR) + { + if (info->lastinx >= 0) + { /* Read only key */ + if (_mi_put_key_in_record(info,(uint) info->lastinx,buf)) + { + my_errno=HA_ERR_CRASHED; + return -1; + } + info->update|= HA_STATE_AKTIV; /* We should find a record */ + return 0; + } + my_errno=HA_ERR_WRONG_INDEX; + } + return(-1); /* Wrong data to read */ +} + + + /* Update auto_increment info */ + +void update_auto_increment(MI_INFO *info,const byte *record) +{ + ulonglong value; + MI_KEYSEG *keyseg=info->s->keyinfo[info->s->base.auto_key-1].seg; + const uchar *key=(uchar*) record+keyseg->start; + + switch (keyseg->type) { + case HA_KEYTYPE_INT8: + case HA_KEYTYPE_BINARY: + value=(ulonglong) *(uchar*) key; + break; + case HA_KEYTYPE_SHORT_INT: + case HA_KEYTYPE_USHORT_INT: + value=(ulonglong) uint2korr(key); + break; + case HA_KEYTYPE_LONG_INT: + case HA_KEYTYPE_ULONG_INT: + value=(ulonglong) uint4korr(key); + break; + case HA_KEYTYPE_INT24: + case HA_KEYTYPE_UINT24: + value=(ulonglong) uint3korr(key); + break; + case HA_KEYTYPE_FLOAT: /* This shouldn't be used */ + { + float f_1; + float4get(f_1,key); + value = (ulonglong) f_1; + break; + } + case HA_KEYTYPE_DOUBLE: /* This shouldn't be used */ + { + double f_1; + float8get(f_1,key); + value = (ulonglong) f_1; + break; + } + case HA_KEYTYPE_LONGLONG: + case HA_KEYTYPE_ULONGLONG: + value= uint8korr(key); + break; + default: + value=0; /* Error */ + break; + } + set_if_bigger(info->s->state.auto_increment,value); +} diff --git a/myisam/mi_locking.c b/myisam/mi_locking.c new file mode 100644 index 00000000000..2ef62c23430 --- /dev/null +++ b/myisam/mi_locking.c @@ -0,0 +1,456 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + locking of isam-tables. + reads info from a isam-table. Must be first request before doing any furter + calls to any isamfunktion. Is used to allow many process use the same + isamdatabase. + */ + +#include "myisamdef.h" +#ifdef __WIN__ +#include <errno.h> +#endif +#if !defined(HAVE_PREAD) && defined(THREAD) +pthread_mutex_t THR_LOCK_keycache; +#endif + + /* lock table by F_UNLCK, F_RDLCK or F_WRLCK */ + +int mi_lock_database(MI_INFO *info, int lock_type) +{ + int error; + uint count; + MYISAM_SHARE *share=info->s; + uint flag; + DBUG_ENTER("mi_lock_database"); + + if (share->options & HA_OPTION_READ_ONLY_DATA || + info->lock_type == lock_type) + DBUG_RETURN(0); + flag=error=0; + pthread_mutex_lock(&share->intern_lock); + if (share->kfile >= 0) /* May only be false on windows */ + { + switch (lock_type) + { + case F_UNLCK: + if (info->lock_type == F_RDLCK) + count= --share->r_locks; + else + count= --share->w_locks; + if (info->lock_type == F_WRLCK && !share->w_locks && + !share->delay_key_write && flush_key_blocks(share->kfile,FLUSH_KEEP)) + { + error=my_errno; + mi_mark_crashed(info); /* Mark that table must be checked */ + } + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + if (end_io_cache(&info->rec_cache)) + { + error=my_errno; + mi_mark_crashed(info); + } + } + if (!count) + { + if (share->changed && !share->w_locks) + { + share->state.process= share->last_process=share->this_process; + share->state.unique= info->last_unique= info->this_unique; +#ifndef HAVE_PREAD + pthread_mutex_lock(&THR_LOCK_keycache); // QQ; Has to be removed! +#endif + if (mi_state_info_write(share->kfile, &share->state, 1)) + error=my_errno; +#ifndef HAVE_PREAD + pthread_mutex_unlock(&THR_LOCK_keycache);// QQ; Has to be removed! +#endif + share->changed=0; + if (myisam_flush) + { +#if defined(__WIN__) + if (_commit(share->kfile)) + error=errno; + if (_commit(info->dfile)) + error=errno; +#elif defined(HAVE_FDATASYNC) + if (fdatasync(share->kfile)) + error=errno; + if (fdatasync(share->dfile)) + error=errno; +#elif defined(HAVE_FSYNC) + if (fsync(share->kfile)) + error=errno; + if (fsync(share->dfile)) + error=errno; +#endif + } + else + share->not_flushed=1; + if (error) + mi_mark_crashed(info); + } + if (share->r_locks) + { /* Only read locks left */ + flag=1; + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + error=my_errno; + } + else if (!share->w_locks) + { /* No more locks */ + flag=1; + if (my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + error=my_errno; + } + } + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + info->lock_type= F_UNLCK; + break; + case F_RDLCK: + if (info->lock_type == F_WRLCK) + { /* Change RW to READONLY */ + if (share->w_locks == 1) + { + flag=1; + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE))) + { + error=my_errno; + break; + } + } + share->w_locks--; + share->r_locks++; + info->lock_type=lock_type; + break; + } + if (!share->r_locks && !share->w_locks) + { + flag=1; + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE)) + { + error=my_errno; + break; + } + if (mi_state_info_read_dsk(share->kfile, &share->state, 1)) + { + error=my_errno; + VOID(my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE))); + my_errno=error; + break; + } + } + VOID(_mi_test_if_changed(info)); + share->r_locks++; + info->lock_type=lock_type; + break; + case F_WRLCK: + if (info->lock_type == F_RDLCK) + { /* Change READONLY to RW */ + if (share->r_locks == 1) + { + flag=1; + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + MYF(info->lock_wait | MY_SEEK_NOT_DONE))) + { + error=my_errno; + break; + } + share->r_locks--; + share->w_locks++; + info->lock_type=lock_type; + break; + } + } + if (!(share->options & HA_OPTION_READ_ONLY_DATA)) + { + if (!share->w_locks) + { + flag=1; + VOID(my_seek(share->kfile,0L,MY_SEEK_SET,MYF(0))); + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF,info->lock_wait)) + { + error=my_errno; + break; + } + if (!share->r_locks) + { + if (mi_state_info_read_dsk(share->kfile, &share->state, 0)) + { + error=my_errno; + VOID(my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF,info->lock_wait)); + my_errno=error; + break; + } + } + } + } + VOID(_mi_test_if_changed(info)); + info->lock_type=lock_type; + share->w_locks++; + break; + default: + break; /* Impossible */ + } + } + pthread_mutex_unlock(&share->intern_lock); +#if defined(FULL_LOG) || defined(_lint) + lock_type|=(int) (flag << 8); /* Set bit to set if real lock */ + myisam_log_command(MI_LOG_LOCK,info,(byte*) &lock_type,sizeof(lock_type), + error); +#endif + DBUG_RETURN(error); +} /* mi_lock_database */ + + +/**************************************************************************** +** The following functions are called by thr_lock() in threaded applications +****************************************************************************/ + +void mi_get_status(void* param) +{ + MI_INFO *info=(MI_INFO*) param; + DBUG_ENTER("mi_get_status"); + DBUG_PRINT("info",("key_file: %ld data_file: %ld", + (long) info->s->state.state.key_file_length, + (long) info->s->state.state.data_file_length)); +#ifndef DBUG_OFF + if (info->state->key_file_length > info->s->state.state.key_file_length || + info->state->data_file_length > info->s->state.state.data_file_length) + DBUG_PRINT("warning",("old info: key_file: %ld data_file: %ld", + (long) info->state->key_file_length, + (long) info->state->data_file_length)); +#endif + info->save_state=info->s->state.state; + info->state= &info->save_state; + DBUG_VOID_RETURN; +} + +void mi_update_status(void* param) +{ + MI_INFO *info=(MI_INFO*) param; + /* + Because someone may have closed the table we point at, we only + update the state if its our own state. This isn't a problem as + we are always pointing at our own lock or at a read lock. + (This is enforced by thr_multi_lock.c) + */ + if (info->state == &info->save_state) + { +#ifndef DBUG_OFF + DBUG_PRINT("info",("updating status: key_file: %ld data_file: %ld", + (long) info->state->key_file_length, + (long) info->state->data_file_length)); + if (info->state->key_file_length < info->s->state.state.key_file_length || + info->state->data_file_length < info->s->state.state.data_file_length) + DBUG_PRINT("warning",("old info: key_file: %ld data_file: %ld", + (long) info->s->state.state.key_file_length, + (long) info->s->state.state.data_file_length)); +#endif + info->s->state.state= *info->state; + info->state= &info->s->state.state; + } + + /* + We have to flush the write cache here as other threads may start + reading the table before mi_lock_database() is called + */ + if (info->opt_flag & WRITE_CACHE_USED) + { + if (end_io_cache(&info->rec_cache)) + { + mi_mark_crashed(info); + } + info->opt_flag&= ~WRITE_CACHE_USED; + } +} + +void mi_copy_status(void* to,void *from) +{ + ((MI_INFO*) to)->state= &((MI_INFO*) from)->save_state; +} + +my_bool mi_check_status(void* param) +{ + MI_INFO *info=(MI_INFO*) param; + return (my_bool) (info->s->state.dellink != HA_OFFSET_ERROR); +} + + +/**************************************************************************** + ** functions to read / write the state +****************************************************************************/ + +int _mi_readinfo(register MI_INFO *info, int lock_type, int check_keybuffer) +{ + MYISAM_SHARE *share; + DBUG_ENTER("_mi_readinfo"); + + share=info->s; + if (info->lock_type == F_UNLCK) + { + if (!share->r_locks && !share->w_locks) + { + if ((info->tmp_lock_type=lock_type) != F_RDLCK) + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE)) + DBUG_RETURN(1); + if (mi_state_info_read_dsk(share->kfile, &share->state, 1)) + { + int error=my_errno ? my_errno : -1; + VOID(my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE))); + my_errno=error; + DBUG_RETURN(1); + } + } + if (check_keybuffer) + VOID(_mi_test_if_changed(info)); + } + else if (lock_type == F_WRLCK && info->lock_type == F_RDLCK) + { + my_errno=EACCES; /* Not allowed to change */ + DBUG_RETURN(-1); /* when have read_lock() */ + } + DBUG_RETURN(0); +} /* _mi_readinfo */ + + + /* Every isam-function that uppdates the isam-database must! end */ + /* with this request */ + /* ARGSUSED */ + +int _mi_writeinfo(register MI_INFO *info, uint operation) +{ + int error,olderror; + MYISAM_SHARE *share; + DBUG_ENTER("_mi_writeinfo"); + + error=0; + share=info->s; + if (share->r_locks == 0 && share->w_locks == 0) + { + olderror=my_errno; /* Remember last error */ + if (operation) + { /* Two threads can't be here */ + share->state.process= share->last_process= share->this_process; + share->state.unique= info->last_unique= info->this_unique; + if ((error=mi_state_info_write(share->kfile, &share->state, 1))) + olderror=my_errno; +#ifdef __WIN__ + if (myisam_flush) + { + _commit(share->kfile); + _commit(info->dfile); + } +#endif + } + if (!(operation & WRITEINFO_NO_UNLOCK) && + my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + DBUG_RETURN(1); + my_errno=olderror; + } + else if (operation) + { + share->changed= 1; /* Mark keyfile changed */ + } + DBUG_RETURN(error); +} /* _mi_writeinfo */ + + + /* Test if someone has changed the database */ + /* (Should be called after readinfo) */ + +int _mi_test_if_changed(register MI_INFO *info) +{ + MYISAM_SHARE *share=info->s; + if (share->state.process != share->last_process || + share->state.unique != info->last_unique) + { /* Keyfile has changed */ + if (share->state.process != share->this_process) + VOID(flush_key_blocks(share->kfile,FLUSH_RELEASE)); + share->last_process=share->state.process; + info->last_unique= share->state.unique; + info->update|= HA_STATE_WRITTEN; /* Must use file on next */ + info->data_changed= 1; /* For mi_is_changed */ + return 1; + } + return (!(info->update & HA_STATE_AKTIV) || + (info->update & (HA_STATE_WRITTEN | HA_STATE_DELETED | + HA_STATE_KEY_CHANGED))); +} /* _mi_test_if_changed */ + + +/* Put a mark in the .ISM file that someone is updating the table */ + +int _mi_mark_file_changed(MI_INFO *info) +{ + char buff[3]; + register MYISAM_SHARE *share=info->s; + if (!share->state.changed || ! share->global_changed) + { + share->state.changed|=1; + if (!share->global_changed) + { + share->global_changed=1; + share->state.open_count++; + } + mi_int2store(buff,share->state.open_count); + buff[2]=1; /* Mark that it's changed */ + return (my_pwrite(share->kfile,buff,sizeof(buff), + sizeof(share->state.header), + MYF(MY_NABP))); + } + return 0; +} + + +/* + This is only called by close or by extra(HA_FLUSH) if the OS has the pwrite() + call. In these context the following code should be safe! + */ + +int _mi_decrement_open_count(MI_INFO *info) +{ + char buff[2]; + register MYISAM_SHARE *share=info->s; + int lock_error=0,write_error=0; + if (share->global_changed) + { + uint old_lock=info->lock_type; + share->global_changed=0; + lock_error=mi_lock_database(info,F_WRLCK); + /* Its not fatal even if we couldn't get the lock ! */ + if (share->state.open_count > 0) + { + share->state.open_count--; + mi_int2store(buff,share->state.open_count); + write_error=my_pwrite(share->kfile,buff,sizeof(buff), + sizeof(share->state.header), + MYF(MY_NABP)); + } + if (!lock_error) + lock_error=mi_lock_database(info,old_lock); + } + return test(lock_error || write_error); +} diff --git a/myisam/mi_log.c b/myisam/mi_log.c new file mode 100644 index 00000000000..9f08b835d14 --- /dev/null +++ b/myisam/mi_log.c @@ -0,0 +1,158 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Logging of isamcommands and records on logfile */ + +#include "myisamdef.h" +#if defined(MSDOS) || defined(__WIN__) +#include <errno.h> +#include <fcntl.h> +#ifndef __WIN__ +#include <process.h> +#endif +#endif +#ifdef VMS +#include <processes.h> +#endif + +#undef GETPID /* For HPUX */ +#ifdef THREAD +#define GETPID() (log_type == 1 ? getpid() : (long) my_thread_id()); +#else +#define GETPID() getpid() +#endif + + /* Activate logging if flag is 1 and reset logging if flag is 0 */ + +static int log_type=0; + +int mi_log(int activate_log) +{ + int error=0; + char buff[FN_REFLEN]; + DBUG_ENTER("mi_log"); + + log_type=activate_log; + if (activate_log) + { + if (myisam_log_file < 0) + { + if ((myisam_log_file = my_create(fn_format(buff,myisam_log_filename, + "",".log",4), + 0,(O_RDWR | O_BINARY | O_APPEND),MYF(0))) + < 0) + DBUG_RETURN(my_errno); + } + } + else if (myisam_log_file >= 0) + { + error=my_close(myisam_log_file,MYF(0)) ? my_errno : 0 ; + myisam_log_file= -1; + } + DBUG_RETURN(error); +} + + + /* Logging of records and commands on logfile */ + /* All logs starts with command(1) dfile(2) process(4) result(2) */ + +void _myisam_log(enum myisam_log_commands command, MI_INFO *info, const byte *buffert, uint length) +{ + char buff[11]; + int error,old_errno; + ulong pid=(ulong) GETPID(); + old_errno=my_errno; + bzero(buff,sizeof(buff)); + buff[0]=(char) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+9,length); + + pthread_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + VOID(my_write(myisam_log_file,buff,sizeof(buff),MYF(0))); + VOID(my_write(myisam_log_file,buffert,length,MYF(0))); + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + pthread_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} + + +void _myisam_log_command(enum myisam_log_commands command, MI_INFO *info, + const byte *buffert, uint length, int result) +{ + char buff[9]; + int error,old_errno; + ulong pid=(ulong) GETPID(); + + old_errno=my_errno; + buff[0]=(char) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+7,result); + pthread_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + VOID(my_write(myisam_log_file,buff,sizeof(buff),MYF(0))); + if (buffert) + VOID(my_write(myisam_log_file,buffert,length,MYF(0))); + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + pthread_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} + + +void _myisam_log_record(enum myisam_log_commands command, MI_INFO *info, + const byte *record, my_off_t filepos, int result) +{ + char buff[21],*pos; + int error,old_errno; + uint length; + ulong pid=(ulong) GETPID(); + + old_errno=my_errno; + if (!info->s->base.blobs) + length=info->s->base.reclength; + else + length=info->s->base.reclength+ _my_calc_total_blob_length(info,record); + buff[0]=(char) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+7,result); + mi_sizestore(buff+9,filepos); + mi_int4store(buff+17,length); + pthread_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + VOID(my_write(myisam_log_file,buff,sizeof(buff),MYF(0))); + VOID(my_write(myisam_log_file,(byte*) record,info->s->base.reclength,MYF(0))); + if (info->s->base.blobs) + { + MI_BLOB *blob,*end; + + for (end=info->blobs+info->s->base.blobs, blob= info->blobs; + blob != end ; + blob++) + { + memcpy_fixed(&pos,record+blob->offset+blob->pack_length,sizeof(char*)); + VOID(my_write(myisam_log_file,pos,blob->length,MYF(0))); + } + } + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + pthread_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} diff --git a/myisam/mi_open.c b/myisam/mi_open.c new file mode 100644 index 00000000000..2067e343246 --- /dev/null +++ b/myisam/mi_open.c @@ -0,0 +1,994 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* open a isam-database */ + +#include "fulltext.h" +#include <m_ctype.h> + +#if defined(MSDOS) || defined(__WIN__) +#ifdef __WIN__ +#include <fcntl.h> +#else +#include <process.h> /* Prototype for getpid */ +#endif +#endif +#ifdef VMS +#include "static.c" +#endif + +static void setup_functions(MYISAM_SHARE *info); +static void setup_key_functions(MI_KEYDEF *keyinfo); +#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \ + pos+=size;} + + +/****************************************************************************** +** Return the shared struct if the table is already open. +** In MySQL the server will handle version issues. +******************************************************************************/ + +static MI_INFO *test_if_reopen(char *filename) +{ + LIST *pos; + + for (pos=myisam_open_list ; pos ; pos=pos->next) + { + MI_INFO *info=(MI_INFO*) pos->data; + MYISAM_SHARE *share=info->s; + if (!strcmp(share->filename,filename) && share->last_version) + return info; + } + return 0; +} + + +/****************************************************************************** + open a isam database. + if handle_locking is 0 then exit with error if database is locked + if handle_locking is 1 then wait if database is locked + if handle_locking is 2 then continue, but count-vars in st_i_info + may be wrong. count-vars are automaticly fixed after next isam + request. +******************************************************************************/ + + +MI_INFO *mi_open(const char *name, int mode, uint handle_locking) +{ + int lock_error,kfile,open_mode,save_errno; + uint i,j,len,errpos,head_length,base_pos,offset,info_length,extra,keys, + key_parts,unique_key_parts,tmp_length,uniques; + char name_buff[FN_REFLEN],*disk_cache,*disk_pos; + MI_INFO info,*m_info,*old_info; + MYISAM_SHARE share_buff,*share; + ulong rec_per_key_part[MI_MAX_POSSIBLE_KEY*MI_MAX_KEY_SEG]; + my_off_t key_root[MI_MAX_POSSIBLE_KEY],key_del[MI_MAX_KEY_BLOCK_SIZE]; + ulonglong max_key_file_length, max_data_file_length; + DBUG_ENTER("mi_open"); + + LINT_INIT(m_info); + kfile= -1; + lock_error=1; + errpos=0; + head_length=sizeof(share_buff.state.header); + bzero((byte*) &info,sizeof(info)); + + VOID(fn_format(name_buff,name,"",MI_NAME_IEXT,4+16+32)); + pthread_mutex_lock(&THR_LOCK_myisam); + if (!(old_info=test_if_reopen(name_buff))) + { + share= &share_buff; + bzero((gptr) &share_buff,sizeof(share_buff)); + share_buff.state.rec_per_key_part=rec_per_key_part; + share_buff.state.key_root=key_root; + share_buff.state.key_del=key_del; + + if ((kfile=my_open(name_buff,(open_mode=O_RDWR) | O_SHARE,MYF(0))) < 0) + { + if ((errno != EROFS && errno != EACCES) || + mode != O_RDONLY || + (kfile=my_open(name_buff,(open_mode=O_RDONLY) | O_SHARE,MYF(0))) < 0) + goto err; + } + errpos=1; + if (my_read(kfile,(char*) share->state.header.file_version,head_length, + MYF(MY_NABP))) + goto err; + + if (memcmp((byte*) share->state.header.file_version, + (byte*) myisam_file_magic, 4)) + { + DBUG_PRINT("error",("Wrong header in %s",name_buff)); + DBUG_DUMP("error_dump",(char*) share->state.header.file_version, + head_length); + my_errno=HA_ERR_CRASHED; + goto err; + } + share->options= mi_uint2korr(share->state.header.options); + if (share->options & + ~(HA_OPTION_PACK_RECORD | HA_OPTION_PACK_KEYS | + HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA | + HA_OPTION_TEMP_COMPRESS_RECORD | HA_OPTION_CHECKSUM | + HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE)) + { + DBUG_PRINT("error",("wrong options: 0x%lx", + share->options)); + my_errno=HA_ERR_OLD_FILE; + goto err; + } + info_length=mi_uint2korr(share->state.header.header_length); + base_pos=mi_uint2korr(share->state.header.base_pos); + if (!(disk_cache=(char*) my_alloca(info_length))) + { + my_errno=ENOMEM; + goto err; + } + errpos=2; + + VOID(my_seek(kfile,0L,MY_SEEK_SET,MYF(0))); + if (!(handle_locking & HA_OPEN_TMP_TABLE)) + { + if ((lock_error=my_lock(kfile,F_RDLCK,0L,F_TO_EOF, + MYF(handle_locking & HA_OPEN_WAIT_IF_LOCKED ? + 0 : MY_DONT_WAIT))) && + !(handle_locking & HA_OPEN_IGNORE_IF_LOCKED)) + goto err; + } + errpos=3; + if (my_read(kfile,disk_cache,info_length,MYF(MY_NABP))) + goto err; + len=mi_uint2korr(share->state.header.state_info_length); + keys= (uint) share->state.header.keys; + uniques= (uint) share->state.header.uniques; + key_parts= mi_uint2korr(share->state.header.key_parts); + unique_key_parts= mi_uint2korr(share->state.header.unique_key_parts); + tmp_length=(MI_STATE_INFO_SIZE + keys * MI_STATE_KEY_SIZE + + key_parts*MI_STATE_KEYSEG_SIZE + + share->state.header.max_block_size*MI_STATE_KEYBLOCK_SIZE); + if (len != MI_STATE_INFO_SIZE) + { + DBUG_PRINT("warning", + ("saved_state_info_length: %d state_info_length: %d", + len,MI_STATE_INFO_SIZE)); + } + share->state_diff_length=len-MI_STATE_INFO_SIZE; + + mi_state_info_read(disk_cache, &share->state); + len= mi_uint2korr(share->state.header.base_info_length); + if (len != MI_BASE_INFO_SIZE) + { + DBUG_PRINT("warning",("saved_base_info_length: %d base_info_length: %d", + len,MI_BASE_INFO_SIZE)) + } + disk_pos=my_n_base_info_read(disk_cache+base_pos, &share->base); + share->state.state_length=base_pos; + + /* Correct max_file_length based on length of sizeof_t */ + max_data_file_length= + (share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? + (((ulonglong) 1 << (share->base.rec_reflength*8))-1) : + (mi_safe_mul(share->base.reclength, + (ulonglong) 1 << (share->base.rec_reflength*8))-1); + max_key_file_length= + mi_safe_mul(MI_KEY_BLOCK_LENGTH, + ((ulonglong) 1 << (share->base.key_reflength*8))-1); +#if SIZEOF_OFF_T == 4 + set_if_smaller(max_data_file_length, INT_MAX32); + set_if_smaller(max_key_file_length, INT_MAX32); +#endif +#if USE_RAID && SYSTEM_SIZEOF_OFF_T == 4 + set_if_smaller(max_key_file_length, INT_MAX32); + if (!share->base.raid_type) + { + set_if_smaller(max_data_file_length, INT_MAX32); + } + else + { + set_if_smaller(max_data_file_length, + (ulonglong) share->base.raid_chunks << 31); + } +#elif !defined(USE_RAID) + if (share->base.raid_type) + { + my_errno=HA_ERR_UNSUPPORTED; + goto err; + } +#endif + share->base.max_data_file_length=(my_off_t) max_data_file_length; + share->base.max_key_file_length=(my_off_t) max_key_file_length; + + if (share->base.max_key_length > MI_MAX_KEY_BUFF || keys > MI_MAX_KEY || + key_parts >= MI_MAX_KEY * MI_MAX_KEY_SEG) + { + my_errno=HA_ERR_UNSUPPORTED; + goto err; + } + if (share->options & HA_OPTION_COMPRESS_RECORD) + share->base.max_key_length+=2; /* For safety */ + + if (!my_multi_malloc(MY_WME, + &share,sizeof(*share), + &share->state.rec_per_key_part,sizeof(long)*key_parts, + &share->keyinfo,keys*sizeof(MI_KEYDEF), + &share->uniqueinfo,uniques*sizeof(MI_UNIQUEDEF), + &share->keyparts, + (key_parts+unique_key_parts+keys+uniques) * + sizeof(MI_KEYSEG), + &share->rec, + (share->base.fields+1)*sizeof(MI_COLUMNDEF), + &share->blobs,sizeof(MI_BLOB)*share->base.blobs, + &share->filename,strlen(name_buff)+1, + &share->state.key_root,keys*sizeof(my_off_t), + &share->state.key_del, + (share->state.header.max_block_size*sizeof(my_off_t)), +#ifdef THREAD + &share->key_root_lock,sizeof(rw_lock_t)*keys, +#endif + NullS)) + goto err; + errpos=4; + *share=share_buff; + memcpy((char*) share->state.rec_per_key_part, + (char*) rec_per_key_part, sizeof(long)*key_parts); + memcpy((char*) share->state.key_root, + (char*) key_root, sizeof(my_off_t)*keys); + memcpy((char*) share->state.key_del, + (char*) key_del, (sizeof(my_off_t) * + share->state.header.max_block_size)); + strmov(share->filename,name_buff); + + share->blocksize=min(IO_SIZE,myisam_block_size); + { + MI_KEYSEG *pos=share->keyparts; + for (i=0 ; i < keys ; i++) + { + disk_pos=mi_keydef_read(disk_pos, &share->keyinfo[i]); + set_if_smaller(share->blocksize,share->keyinfo[i].block_length); + share->keyinfo[i].seg=pos; + for (j=0 ; j < share->keyinfo[i].keysegs; j++,pos++) + { + disk_pos=mi_keyseg_read(disk_pos, pos); + if (pos->type == HA_KEYTYPE_TEXT || pos->type == HA_KEYTYPE_VARTEXT) + { + if (!pos->language) + pos->charset=default_charset_info; + else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME)))) + { + my_errno=HA_ERR_UNKNOWN_CHARSET; + goto err; + } + } + } + if (share->keyinfo[i].flag & HA_FULLTEXT) /* SerG */ + share->keyinfo[i].seg=pos-FT_SEGS; /* SerG */ + share->keyinfo[i].end=pos; + pos->type=HA_KEYTYPE_END; /* End */ + pos->length=share->base.rec_reflength; + pos->null_bit=0; + pos++; + } + for (i=0 ; i < uniques ; i++) + { + disk_pos=mi_uniquedef_read(disk_pos, &share->uniqueinfo[i]); + share->uniqueinfo[i].seg=pos; + for (j=0 ; j < share->uniqueinfo[i].keysegs; j++,pos++) + { + disk_pos=mi_keyseg_read(disk_pos, pos); + if (pos->type == HA_KEYTYPE_TEXT || pos->type == HA_KEYTYPE_VARTEXT) + { + if (!pos->language) + pos->charset=default_charset_info; + else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME)))) + { + my_errno=HA_ERR_UNKNOWN_CHARSET; + goto err; + } + } + } + share->uniqueinfo[i].end=pos; + pos->type=HA_KEYTYPE_END; /* End */ + pos->null_bit=0; + pos++; + } + } + for (i=0 ; i < keys ; i++) + setup_key_functions(share->keyinfo+i); + + for (i=j=offset=0 ; i < share->base.fields ; i++) + { + disk_pos=mi_recinfo_read(disk_pos,&share->rec[i]); + share->rec[i].pack_type=0; + share->rec[i].huff_tree=0; + share->rec[i].offset=offset; + if (share->rec[i].type == (int) FIELD_BLOB) + { + share->blobs[j].pack_length= + share->rec[i].length-mi_portable_sizeof_char_ptr;; + share->blobs[j].offset=offset; + j++; + } + offset+=share->rec[i].length; + } + share->rec[i].type=(int) FIELD_LAST; /* End marker */ + + if (! lock_error) + { + VOID(my_lock(kfile,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE))); + lock_error=1; /* Database unlocked */ + } + +#ifdef USE_RAID + if (share->base.raid_type) + { + if ((info.dfile=my_raid_open(fn_format(name_buff,name,"",MI_NAME_DEXT, + 2+4), + mode | O_SHARE, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(MY_WME | MY_RAID))) < 0) + goto err; + } + else +#endif + if ((info.dfile=my_open(fn_format(name_buff,name,"",MI_NAME_DEXT,2+4), + mode | O_SHARE, + MYF(MY_WME))) < 0) + goto err; + errpos=5; + + share->kfile=kfile; + share->mode=open_mode; + share->this_process=(ulong) getpid(); + share->rnd= (int) share->this_process; /* rnd-counter for splits */ +#ifndef DBUG_OFF + share->rnd=0; /* To make things repeatable */ +#endif + share->last_process= share->state.process; + share->base.key_parts=key_parts; + share->base.all_key_parts=key_parts+unique_key_parts; + if (!(share->last_version=share->state.version)) + share->last_version=1; /* Safety */ + share->rec_reflength=share->base.rec_reflength; /* May be changed */ + share->base.margin_key_file_length=(share->base.max_key_file_length - + (keys ? MI_INDEX_BLOCK_MARGIN * + share->blocksize * keys : 0)); + share->blocksize=min(IO_SIZE,myisam_block_size); + + share->data_file_type=STATIC_RECORD; + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + share->data_file_type = COMPRESSED_RECORD; + share->options|= HA_OPTION_READ_ONLY_DATA; + info.s=share; + if (_mi_read_pack_info(&info, + (pbool) + test(!(share->options & + (HA_OPTION_PACK_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD))))) + goto err; + } + else if (share->options & HA_OPTION_PACK_RECORD) + share->data_file_type = DYNAMIC_RECORD; + my_afree((gptr) disk_cache); + setup_functions(share); +#ifdef THREAD + thr_lock_init(&share->lock); + VOID(pthread_mutex_init(&share->intern_lock,NULL)); + for (i=0; i<keys; i++) + VOID(my_rwlock_init(&share->key_root_lock[i], NULL)); + if (!thr_lock_inited) + { + /* Probably a single threaded program; Don't use concurrent inserts */ + myisam_concurrent_insert=0; + } + else if (myisam_concurrent_insert) + { + share->concurrent_insert= + ((share->options & (HA_OPTION_READ_ONLY_DATA | HA_OPTION_TMP_TABLE | + HA_OPTION_COMPRESS_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD)) || + (handle_locking & HA_OPEN_TMP_TABLE)) ? 0 : 1; + if (share->concurrent_insert) + { + share->lock.get_status=mi_get_status; + share->lock.copy_status=mi_copy_status; + share->lock.update_status=mi_update_status; + share->lock.check_status=mi_check_status; + } + } +#endif + } + else + { + share= old_info->s; + if (mode == O_RDWR && share->mode == O_RDONLY) + { + my_errno=EACCES; /* Can't open in write mode */ + goto err; + } +#ifdef USE_RAID + if (share->base.raid_type) + { + if ((info.dfile=my_raid_open(fn_format(name_buff,old_info->filename,"", + MI_NAME_DEXT, 2+4), + mode | O_SHARE, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(MY_WME | MY_RAID))) < 0) + goto err; + } + else +#endif + if ((info.dfile=my_open(fn_format(name_buff,old_info->filename,"", + MI_NAME_DEXT,2+4), + mode | O_SHARE,MYF(MY_WME))) < 0) + { + my_errno=errno; + goto err; + } + errpos=5; + } + + /* alloc and set up private structure parts */ + if (!my_multi_malloc(MY_WME, + &m_info,sizeof(MI_INFO), + &info.blobs,sizeof(MI_BLOB)*share->base.blobs, + &info.buff,(share->base.max_key_block_length*2+ + share->base.max_key_length), + &info.lastkey,share->base.max_key_length*3+1, + &info.filename,strlen(name)+1, + NullS)) + goto err; + errpos=6; + + strmov(info.filename,name); + memcpy(info.blobs,share->blobs,sizeof(MI_BLOB)*share->base.blobs); + info.lastkey2=info.lastkey+share->base.max_key_length; + + info.s=share; + info.lastpos= HA_OFFSET_ERROR; + info.update= (short) (HA_STATE_NEXT_FOUND+HA_STATE_PREV_FOUND); + info.opt_flag=READ_CHECK_USED; + info.this_unique= (ulong) info.dfile; /* Uniq number in process */ + if (share->data_file_type == COMPRESSED_RECORD) + info.this_unique= share->state.unique; + info.this_loop=0; /* Update counter */ + info.last_unique= share->state.unique; + if (mode == O_RDONLY) + share->options|=HA_OPTION_READ_ONLY_DATA; + info.lock_type=F_UNLCK; + info.quick_mode=0; + info.errkey= -1; + info.page_changed=1; + pthread_mutex_lock(&share->intern_lock); + info.read_record=share->read_record; + share->reopen++; + share->write_flag=MYF(MY_NABP | MY_WAIT_IF_FULL); + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + info.lock_type=F_RDLCK; + share->r_locks++; + } + if ((handle_locking & HA_OPEN_TMP_TABLE) || + (share->options & HA_OPTION_TMP_TABLE)) + { + share->temporary=share->delay_key_write=1; + share->write_flag=MYF(MY_NABP); + share->w_locks++; /* We don't have to update status */ + info.lock_type=F_WRLCK; + } + if (((handle_locking & HA_OPEN_DELAY_KEY_WRITE) || + (share->options & HA_OPTION_DELAY_KEY_WRITE)) && + myisam_delay_key_write) + share->delay_key_write=1; + info.state= &share->state.state; /* Change global values by default */ + pthread_mutex_unlock(&share->intern_lock); + + /* Allocate buffer for one record */ + + extra=0; + if (share->options & HA_OPTION_PACK_RECORD) + extra=ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER; + + tmp_length=max(share->base.pack_reclength,share->base.max_key_length); + info.alloced_rec_buff_length=tmp_length; + if (!(info.rec_alloc=(byte*) my_malloc(tmp_length+extra+8, + MYF(MY_WME | MY_ZEROFILL)))) + goto err; + if (extra) + info.rec_buff=info.rec_alloc+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER); + else + info.rec_buff=info.rec_alloc; + + *m_info=info; +#ifdef THREAD + thr_lock_data_init(&share->lock,&m_info->lock,(void*) m_info); +#endif + m_info->open_list.data=(void*) m_info; + myisam_open_list=list_add(myisam_open_list,&m_info->open_list); + + pthread_mutex_unlock(&THR_LOCK_myisam); + myisam_log(MI_LOG_OPEN,m_info,share->filename,(uint) strlen(share->filename)); + DBUG_RETURN(m_info); + +err: + save_errno=my_errno ? my_errno : HA_ERR_END_OF_FILE; + switch (errpos) { + case 6: + my_free((gptr) m_info,MYF(0)); + /* fall through */ + case 5: + VOID(my_close(info.dfile,MYF(0))); + if (old_info) + break; /* Don't remove open table */ + /* fall through */ + case 4: + my_free((gptr) share,MYF(0)); + /* fall through */ + case 3: + if (! lock_error) + VOID(my_lock(kfile, F_UNLCK, 0L, F_TO_EOF, MYF(MY_SEEK_NOT_DONE))); + /* fall through */ + case 2: + my_afree((gptr) disk_cache); + /* fall through */ + case 1: + VOID(my_close(kfile,MYF(0))); + /* fall through */ + case 0: + default: + break; + } + pthread_mutex_unlock(&THR_LOCK_myisam); + my_errno=save_errno; + DBUG_RETURN (NULL); +} /* mi_open */ + + +ulonglong mi_safe_mul(ulonglong a, ulonglong b) +{ + ulonglong max_val= ~ (ulonglong) 0; /* my_off_t is unsigned */ + + if (!a || max_val / a < b) + return max_val; + return a*b; +} + + /* Set up functions in structs */ + +static void setup_functions(register MYISAM_SHARE *share) +{ + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + share->read_record=_mi_read_pack_record; + share->read_rnd=_mi_read_rnd_pack_record; + if (!(share->options & HA_OPTION_TEMP_COMPRESS_RECORD)) + share->calc_checksum=0; /* No checksum */ + else if (share->options & HA_OPTION_PACK_RECORD) + share->calc_checksum= mi_checksum; + else + share->calc_checksum= mi_static_checksum; + } + else if (share->options & HA_OPTION_PACK_RECORD) + { + share->read_record=_mi_read_dynamic_record; + share->read_rnd=_mi_read_rnd_dynamic_record; + share->delete_record=_mi_delete_dynamic_record; + share->compare_record=_mi_cmp_dynamic_record; + share->compare_unique=_mi_cmp_dynamic_unique; + share->calc_checksum= mi_checksum; + + if (share->base.blobs) + { + share->update_record=_mi_update_blob_record; + share->write_record=_mi_write_blob_record; + } + else + { + share->write_record=_mi_write_dynamic_record; + share->update_record=_mi_update_dynamic_record; + } + } + else + { + share->read_record=_mi_read_static_record; + share->read_rnd=_mi_read_rnd_static_record; + share->delete_record=_mi_delete_static_record; + share->compare_record=_mi_cmp_static_record; + share->update_record=_mi_update_static_record; + share->write_record=_mi_write_static_record; + share->compare_unique=_mi_cmp_static_unique; + share->calc_checksum= mi_static_checksum; + } + if (!(share->options & HA_OPTION_CHECKSUM)) + share->calc_checksum=0; + return; +} + + +static void setup_key_functions(register MI_KEYDEF *keyinfo) +{ + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { /* Simple prefix compression */ + keyinfo->bin_search=_mi_seq_search; + keyinfo->get_key=_mi_get_binary_pack_key; + keyinfo->pack_key=_mi_calc_bin_pack_key_length; + keyinfo->store_key=_mi_store_bin_pack_key; + } + else if (keyinfo->flag & HA_VAR_LENGTH_KEY) + { + keyinfo->bin_search=_mi_seq_search; + keyinfo->get_key= _mi_get_pack_key; + if (keyinfo->seg[0].flag & HA_PACK_KEY) + { /* Prefix compression */ + keyinfo->pack_key=_mi_calc_var_pack_key_length; + keyinfo->store_key=_mi_store_var_pack_key; + } + else + { + keyinfo->pack_key=_mi_calc_var_key_length; /* Variable length key */ + keyinfo->store_key=_mi_store_static_key; + } + } + else + { + keyinfo->bin_search=_mi_bin_search; + keyinfo->get_key=_mi_get_static_key; + keyinfo->pack_key=_mi_calc_static_key_length; + keyinfo->store_key=_mi_store_static_key; + } + return; +} + + +/*************************************************************************** +** Function to save and store the header in the index file (.MSI) +***************************************************************************/ + +uint mi_state_info_write(File file, MI_STATE_INFO *state, uint pWrite) +{ + uchar buff[MI_STATE_INFO_SIZE + MI_STATE_EXTRA_SIZE]; + uchar *ptr=buff; + uint i, keys= (uint) state->header.keys, + key_blocks=state->header.max_block_size; + + memcpy_fixed(ptr,&state->header,sizeof(state->header)); + ptr+=sizeof(state->header); + + /* open_count must be first because of _mi_mark_file_changed ! */ + mi_int2store(ptr,state->open_count); ptr +=2; + *ptr++= (uchar)state->changed; *ptr++= state->sortkey; + mi_rowstore(ptr,state->state.records); ptr +=8; + mi_rowstore(ptr,state->state.del); ptr +=8; + mi_rowstore(ptr,state->split); ptr +=8; + mi_sizestore(ptr,state->dellink); ptr +=8; + mi_sizestore(ptr,state->state.key_file_length); ptr +=8; + mi_sizestore(ptr,state->state.data_file_length); ptr +=8; + mi_sizestore(ptr,state->state.empty); ptr +=8; + mi_sizestore(ptr,state->state.key_empty); ptr +=8; + mi_int8store(ptr,state->auto_increment); ptr +=8; + mi_int8store(ptr,(ulonglong) state->checksum);ptr +=8; + mi_int4store(ptr,state->process); ptr +=4; + mi_int4store(ptr,state->unique); ptr +=4; + mi_int4store(ptr,state->status); ptr +=4; + *ptr++=0; *ptr++=0; *ptr++=0; *ptr++=0; /* extra */ + + ptr+=state->state_diff_length; + + for (i=0; i < keys; i++) + { + mi_sizestore(ptr,state->key_root[i]); ptr +=8; + } + for (i=0; i < key_blocks; i++) + { + mi_sizestore(ptr,state->key_del[i]); ptr +=8; + } + if (pWrite & 2) /* From isamchk */ + { + uint key_parts= mi_uint2korr(state->header.key_parts); + mi_int4store(ptr,state->sec_index_changed); ptr +=4; + mi_int4store(ptr,state->sec_index_used); ptr +=4; + mi_int4store(ptr,state->version); ptr +=4; + mi_int8store(ptr,state->key_map); ptr +=8; + mi_int8store(ptr,(ulonglong) state->create_time); ptr +=8; + mi_int8store(ptr,(ulonglong) state->recover_time); ptr +=8; + mi_int8store(ptr,(ulonglong) state->check_time); ptr +=8; + mi_sizestore(ptr,state->rec_per_key_rows); ptr+=8; + for (i=0 ; i < key_parts ; i++) + { + mi_int4store(ptr,state->rec_per_key_part[i]); ptr+=4; + } + } + + if (pWrite & 1) + return my_pwrite(file,(char*) buff, (uint) (ptr-buff), 0L, + MYF(MY_NABP | MY_THREADSAFE)); + else + return my_write(file, (char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + + +char *mi_state_info_read(char *ptr, MI_STATE_INFO *state) +{ + uint i,keys,key_parts,key_blocks; + memcpy_fixed(&state->header,ptr, sizeof(state->header)); + ptr +=sizeof(state->header); + keys=(uint) state->header.keys; + key_parts=mi_uint2korr(state->header.key_parts); + key_blocks=state->header.max_block_size; + + state->open_count = mi_uint2korr(ptr); ptr +=2; + state->changed= (bool) *ptr++; + state->sortkey = (uint) *ptr++; + state->state.records= mi_rowkorr(ptr); ptr +=8; + state->state.del = mi_rowkorr(ptr); ptr +=8; + state->split = mi_rowkorr(ptr); ptr +=8; + state->dellink= mi_sizekorr(ptr); ptr +=8; + state->state.key_file_length = mi_sizekorr(ptr); ptr +=8; + state->state.data_file_length= mi_sizekorr(ptr); ptr +=8; + state->state.empty = mi_sizekorr(ptr); ptr +=8; + state->state.key_empty= mi_sizekorr(ptr); ptr +=8; + state->auto_increment=mi_uint8korr(ptr); ptr +=8; + state->checksum=(ha_checksum) mi_uint8korr(ptr); ptr +=8; + state->process= mi_uint4korr(ptr); ptr +=4; + state->unique = mi_uint4korr(ptr); ptr +=4; + state->status = mi_uint4korr(ptr); ptr +=4; + ptr +=4; /* extra */ + for (i=0; i < keys; i++) + { + state->key_root[i]= mi_sizekorr(ptr); ptr +=8; + } + for (i=0; i < key_blocks; i++) + { + state->key_del[i] = mi_sizekorr(ptr); ptr +=8; + } + ptr+= state->state_diff_length; + state->sec_index_changed = mi_uint4korr(ptr); ptr +=4; + state->sec_index_used = mi_uint4korr(ptr); ptr +=4; + state->version = mi_uint4korr(ptr); ptr +=4; + state->key_map = mi_uint8korr(ptr); ptr +=8; + state->create_time = (time_t) mi_sizekorr(ptr); ptr +=8; + state->recover_time =(time_t) mi_sizekorr(ptr); ptr +=8; + state->check_time = (time_t) mi_sizekorr(ptr); ptr +=8; + state->rec_per_key_rows=mi_sizekorr(ptr); ptr +=8; + for (i=0 ; i < key_parts ; i++) + { + state->rec_per_key_part[i]= mi_uint4korr(ptr); ptr+=4; + } + return ptr; +} + + +uint mi_state_info_read_dsk(File file, MI_STATE_INFO *state, my_bool pRead) +{ + char buff[MI_STATE_INFO_SIZE + MI_STATE_EXTRA_SIZE]; + + if (pRead) + { + if (my_pread(file, buff, state->state_length,0L, MYF(MY_NABP))) + return (MY_FILE_ERROR); + } + else if (my_read(file, buff, state->state_length,MYF(MY_NABP))) + return (MY_FILE_ERROR); + mi_state_info_read(buff, state); + return 0; +} + + +/**************************************************************************** +** store and read of MI_BASE_INFO +****************************************************************************/ + +uint mi_base_info_write(File file, MI_BASE_INFO *base) +{ + uchar buff[MI_BASE_INFO_SIZE], *ptr=buff; + + mi_sizestore(ptr,base->keystart); ptr +=8; + mi_sizestore(ptr,base->max_data_file_length); ptr +=8; + mi_sizestore(ptr,base->max_key_file_length); ptr +=8; + mi_rowstore(ptr,base->records); ptr +=8; + mi_rowstore(ptr,base->reloc); ptr +=8; + mi_int4store(ptr,base->mean_row_length); ptr +=4; + mi_int4store(ptr,base->reclength); ptr +=4; + mi_int4store(ptr,base->pack_reclength); ptr +=4; + mi_int4store(ptr,base->min_pack_length); ptr +=4; + mi_int4store(ptr,base->max_pack_length); ptr +=4; + mi_int4store(ptr,base->min_block_length); ptr +=4; + mi_int4store(ptr,base->fields); ptr +=4; + mi_int4store(ptr,base->pack_fields); ptr +=4; + *ptr++=base->rec_reflength; + *ptr++=base->key_reflength; + *ptr++=base->keys; + *ptr++=base->auto_key; + mi_int2store(ptr,base->pack_bits); ptr +=2; + mi_int2store(ptr,base->blobs); ptr +=2; + mi_int2store(ptr,base->max_key_block_length); ptr +=2; + mi_int2store(ptr,base->max_key_length); ptr +=2; + mi_int2store(ptr,base->extra_alloc_bytes); ptr +=2; + *ptr++= base->extra_alloc_procent; + *ptr++= base->raid_type; + mi_int2store(ptr,base->raid_chunks); ptr +=2; + mi_int4store(ptr,base->raid_chunksize); ptr +=4; + bzero(ptr,6); ptr +=6; /* extra */ + return my_write(file,(char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + + +char *my_n_base_info_read(char *ptr, MI_BASE_INFO *base) +{ + base->keystart = mi_sizekorr(ptr); ptr +=8; + base->max_data_file_length = mi_sizekorr(ptr); ptr +=8; + base->max_key_file_length = mi_sizekorr(ptr); ptr +=8; + base->records = (ha_rows) mi_sizekorr(ptr); ptr +=8; + base->reloc = (ha_rows) mi_sizekorr(ptr); ptr +=8; + base->mean_row_length = mi_uint4korr(ptr); ptr +=4; + base->reclength = mi_uint4korr(ptr); ptr +=4; + base->pack_reclength = mi_uint4korr(ptr); ptr +=4; + base->min_pack_length = mi_uint4korr(ptr); ptr +=4; + base->max_pack_length = mi_uint4korr(ptr); ptr +=4; + base->min_block_length = mi_uint4korr(ptr); ptr +=4; + base->fields = mi_uint4korr(ptr); ptr +=4; + base->pack_fields = mi_uint4korr(ptr); ptr +=4; + + base->rec_reflength = *ptr++; + base->key_reflength = *ptr++; + base->keys= *ptr++; + base->auto_key= *ptr++; + base->pack_bits = mi_uint2korr(ptr); ptr +=2; + base->blobs = mi_uint2korr(ptr); ptr +=2; + base->max_key_block_length= mi_uint2korr(ptr); ptr +=2; + base->max_key_length = mi_uint2korr(ptr); ptr +=2; + base->extra_alloc_bytes = mi_uint2korr(ptr); ptr +=2; + base->extra_alloc_procent = *ptr++; + base->raid_type= *ptr++; + base->raid_chunks= mi_uint2korr(ptr); ptr +=2; + base->raid_chunksize= mi_uint4korr(ptr); ptr +=4; + /* TO BE REMOVED: Fix for old RAID files */ + if (base->raid_type == 0) + { + base->raid_chunks=0; + base->raid_chunksize=0; + } + + ptr+=6; + return ptr; +} + +/*-------------------------------------------------------------------------- + mi_keydef +---------------------------------------------------------------------------*/ + +uint mi_keydef_write(File file, MI_KEYDEF *keydef) +{ + uchar buff[MI_KEYDEF_SIZE]; + uchar *ptr=buff; + + *ptr++ = (uchar) keydef->keysegs; + *ptr++ = 0; /* not used */ + mi_int2store(ptr,keydef->flag); ptr +=2; + mi_int2store(ptr,keydef->block_length); ptr +=2; + mi_int2store(ptr,keydef->keylength); ptr +=2; + mi_int2store(ptr,keydef->minlength); ptr +=2; + mi_int2store(ptr,keydef->maxlength); ptr +=2; + return my_write(file,(char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + +char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef) +{ + keydef->keysegs = (uint) *ptr++; + ptr++; + keydef->flag = mi_uint2korr(ptr); ptr +=2; + keydef->block_length = mi_uint2korr(ptr); ptr +=2; + keydef->keylength = mi_uint2korr(ptr); ptr +=2; + keydef->minlength = mi_uint2korr(ptr); ptr +=2; + keydef->maxlength = mi_uint2korr(ptr); ptr +=2; + keydef->block_size = keydef->block_length/MI_KEY_BLOCK_LENGTH-1; + keydef->underflow_block_length=keydef->block_length/3; + keydef->version = 0; /* Not saved */ + return ptr; +} + +/*************************************************************************** +** mi_keyseg +***************************************************************************/ + +int mi_keyseg_write(File file, const MI_KEYSEG *keyseg) +{ + uchar buff[MI_KEYSEG_SIZE]; + uchar *ptr=buff; + + *ptr++ =keyseg->type; + *ptr++ =keyseg->language; + *ptr++ =keyseg->null_bit; + *ptr++ =keyseg->bit_start; + *ptr++ =keyseg->bit_end; + *ptr++ =0; /* Not used */ + mi_int2store(ptr,keyseg->flag); ptr+=2; + mi_int2store(ptr,keyseg->length); ptr+=2; + mi_int4store(ptr,keyseg->start); ptr+=4; + mi_int4store(ptr,keyseg->null_pos); ptr+=4; + + return my_write(file,(char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + + +char *mi_keyseg_read(char *ptr, MI_KEYSEG *keyseg) +{ + keyseg->type = *ptr++; + keyseg->language = *ptr++; + keyseg->null_bit = *ptr++; + keyseg->bit_start = *ptr++; + keyseg->bit_end = *ptr++; + ptr++; + keyseg->flag = mi_uint2korr(ptr); ptr +=2; + keyseg->length = mi_uint2korr(ptr); ptr +=2; + keyseg->start = mi_uint4korr(ptr); ptr +=4; + keyseg->null_pos = mi_uint4korr(ptr); ptr +=4; + keyseg->charset=0; /* Will be filled in later */ + return ptr; +} + +/*-------------------------------------------------------------------------- + mi_uniquedef +---------------------------------------------------------------------------*/ + +uint mi_uniquedef_write(File file, MI_UNIQUEDEF *def) +{ + uchar buff[MI_UNIQUEDEF_SIZE]; + uchar *ptr=buff; + + mi_int2store(ptr,def->keysegs); ptr+=2; + *ptr++= (uchar) def->key; + *ptr++ = (uchar) def->null_are_equal; + + return my_write(file,(char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + +char *mi_uniquedef_read(char *ptr, MI_UNIQUEDEF *def) +{ + def->keysegs = mi_uint2korr(ptr); + def->key = ptr[2]; + def->null_are_equal=ptr[3]; + return ptr+4; /* 1 extra byte */ +} + +/*************************************************************************** +** MI_COLUMNDEF +***************************************************************************/ + +uint mi_recinfo_write(File file, MI_COLUMNDEF *recinfo) +{ + uchar buff[MI_COLUMNDEF_SIZE]; + uchar *ptr=buff; + + mi_int2store(ptr,recinfo->type); ptr +=2; + mi_int2store(ptr,recinfo->length); ptr +=2; + *ptr++ = recinfo->null_bit; + mi_int2store(ptr,recinfo->null_pos); ptr+= 2; + return my_write(file,(char*) buff, (uint) (ptr-buff), MYF(MY_NABP)); +} + +char *mi_recinfo_read(char *ptr, MI_COLUMNDEF *recinfo) +{ + recinfo->type= mi_sint2korr(ptr); ptr +=2; + recinfo->length=mi_uint2korr(ptr); ptr +=2; + recinfo->null_bit= (uint8) *ptr++; + recinfo->null_pos=mi_uint2korr(ptr); ptr +=2; + return ptr; +} diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c new file mode 100644 index 00000000000..b6a9435ee3d --- /dev/null +++ b/myisam/mi_packrec.c @@ -0,0 +1,1312 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + /* Functions to compressed records */ + +#include "myisamdef.h" + +#define IS_CHAR ((uint) 32768) /* Bit if char (not offset) in tree */ + +#if INT_MAX > 65536L +#define BITS_SAVED 32 +#define MAX_QUICK_TABLE_BITS 9 /* Because we may shift in 24 bits */ +#else +#define BITS_SAVED 16 +#define MAX_QUICK_TABLE_BITS 6 +#endif + +#define get_bit(BU) ((BU)->bits ? \ + (BU)->current_byte & ((mi_bit_type) 1 << --(BU)->bits) :\ + (fill_buffer(BU), (BU)->bits= BITS_SAVED-1,\ + (BU)->current_byte & ((mi_bit_type) 1 << (BITS_SAVED-1)))) +#define skipp_to_next_byte(BU) ((BU)->bits&=~7) +#define get_bits(BU,count) (((BU)->bits >= count) ? (((BU)->current_byte >> ((BU)->bits-=count)) & mask[count]) : fill_and_get_bits(BU,count)) + +#define decode_bytes_test_bit(bit) \ + if (low_byte & (1 << (7-bit))) \ + pos++; \ + if (*pos & IS_CHAR) \ + { bits-=(bit+1); break; } \ + pos+= *pos + + +static void read_huff_table(MI_BIT_BUFF *bit_buff,MI_DECODE_TREE *decode_tree, + uint16 **decode_table,byte **intervall_buff, + uint16 *tmp_buff); +static void make_quick_table(uint16 *to_table,uint16 *decode_table, + uint *next_free,uint value,uint bits, + uint max_bits); +static void fill_quick_table(uint16 *table,uint bits, uint max_bits, + uint value); +static uint copy_decode_table(uint16 *to_pos,uint offset, + uint16 *decode_table); +static uint find_longest_bitstream(uint16 *table); +static void (*get_unpack_function(MI_COLUMNDEF *rec))(MI_COLUMNDEF *field, + MI_BIT_BUFF *buff, + uchar *to, + uchar *end); +static void uf_zerofill_skipp_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_skipp_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_normal(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_endspace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_endspace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_endspace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_endspace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_prespace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_prespace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_prespace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_prespace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_zerofill_normal(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_constant(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_intervall(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_blob(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_varchar(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void decode_bytes(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static uint decode_pos(MI_BIT_BUFF *bit_buff,MI_DECODE_TREE *decode_tree); +static void init_bit_buffer(MI_BIT_BUFF *bit_buff,uchar *buffer,uint length); +static uint fill_and_get_bits(MI_BIT_BUFF *bit_buff,uint count); +static void fill_buffer(MI_BIT_BUFF *bit_buff); +static uint max_bit(uint value); +#ifdef HAVE_MMAP +static uchar *_mi_mempack_get_block_info(MI_INFO *myisam,MI_BLOCK_INFO *info, + uchar *header); +#endif + +static mi_bit_type mask[]= +{ + 0x00000000, + 0x00000001, 0x00000003, 0x00000007, 0x0000000f, + 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff, + 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, + 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, +#if BITS_SAVED > 16 + 0x0001ffff, 0x0003ffff, 0x0007ffff, 0x000fffff, + 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff, + 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, + 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff, +#endif + }; + + + /* Read all packed info, allocate memory and fix field structs */ + +my_bool _mi_read_pack_info(MI_INFO *info, pbool fix_keys) +{ + File file; + int diff_length; + uint i,trees,huff_tree_bits,rec_reflength,length; + uint16 *decode_table,*tmp_buff; + ulong elements,intervall_length; + char *disk_cache,*intervall_buff; + uchar header[32]; + MYISAM_SHARE *share=info->s; + MI_BIT_BUFF bit_buff; + DBUG_ENTER("_mi_read_pack_info"); + + if (myisam_quick_table_bits < 4) + myisam_quick_table_bits=4; + else if (myisam_quick_table_bits > MAX_QUICK_TABLE_BITS) + myisam_quick_table_bits=MAX_QUICK_TABLE_BITS; + + file=info->dfile; + my_errno=0; + if (my_read(file,(byte*) header,sizeof(header),MYF(MY_NABP))) + { + if (!my_errno) + my_errno=HA_ERR_END_OF_FILE; + DBUG_RETURN(1); + } + if (memcmp((byte*) header,(byte*) myisam_pack_file_magic,4)) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + share->pack.header_length= uint4korr(header+4); + share->min_pack_length=(uint) uint4korr(header+8); + share->max_pack_length=(uint) uint4korr(header+12); + set_if_bigger(share->base.pack_reclength,share->max_pack_length); + elements=uint4korr(header+16); + intervall_length=uint4korr(header+20); + trees=uint2korr(header+24); + share->pack.ref_length=header[26]; + rec_reflength=header[27]; + diff_length=(int) rec_reflength - (int) share->base.rec_reflength; + if (fix_keys) + share->rec_reflength=rec_reflength; + share->base.min_block_length=share->min_pack_length+share->pack.ref_length; + + if (!(share->decode_trees=(MI_DECODE_TREE*) + my_malloc((uint) (trees*sizeof(MI_DECODE_TREE)+ + intervall_length*sizeof(byte)), + MYF(MY_WME)))) + DBUG_RETURN(1); + intervall_buff=(byte*) (share->decode_trees+trees); + + length=(uint) (elements*2+trees*(1 << myisam_quick_table_bits)); + if (!(share->decode_tables=(uint16*) + my_malloc((length+512)*sizeof(uint16)+ + (uint) (share->pack.header_length+7), + MYF(MY_WME | MY_ZEROFILL)))) + { + my_free((gptr) share->decode_trees,MYF(0)); + DBUG_RETURN(1); + } + tmp_buff=share->decode_tables+length; + disk_cache=(byte*) (tmp_buff+512); + + if (my_read(file,disk_cache, + (uint) (share->pack.header_length-sizeof(header)), + MYF(MY_NABP))) + { + my_free((gptr) share->decode_trees,MYF(0)); + my_free((gptr) share->decode_tables,MYF(0)); + DBUG_RETURN(1); + } + + huff_tree_bits=max_bit(trees ? trees-1 : 0); + init_bit_buffer(&bit_buff, (uchar*) disk_cache, + (uint) (share->pack.header_length-sizeof(header))); + /* Read new info for each field */ + for (i=0 ; i < share->base.fields ; i++) + { + share->rec[i].base_type=(enum en_fieldtype) get_bits(&bit_buff,5); + share->rec[i].pack_type=(uint) get_bits(&bit_buff,6); + share->rec[i].space_length_bits=get_bits(&bit_buff,5); + share->rec[i].huff_tree=share->decode_trees+(uint) get_bits(&bit_buff, + huff_tree_bits); + share->rec[i].unpack=get_unpack_function(share->rec+i); + } + skipp_to_next_byte(&bit_buff); + decode_table=share->decode_tables; + for (i=0 ; i < trees ; i++) + read_huff_table(&bit_buff,share->decode_trees+i,&decode_table, + &intervall_buff,tmp_buff); + decode_table=(uint16*) + my_realloc((gptr) share->decode_tables, + (uint) ((byte*) decode_table - (byte*) share->decode_tables), + MYF(MY_HOLD_ON_ERROR)); + { + long diff=PTR_BYTE_DIFF(decode_table,share->decode_tables); + share->decode_tables=decode_table; + for (i=0 ; i < trees ; i++) + share->decode_trees[i].table=ADD_TO_PTR(share->decode_trees[i].table, + diff, + uint16*); + } + + /* Fix record-ref-length for keys */ + if (fix_keys) + { + for (i=0 ; i < share->base.keys ; i++) + { + share->keyinfo[i].keylength+=(uint16) diff_length; + share->keyinfo[i].minlength+=(uint16) diff_length; + share->keyinfo[i].maxlength+=(uint16) diff_length; + share->keyinfo[i].seg[share->keyinfo[i].keysegs].length= + (uint16) rec_reflength; + } + } + + if (bit_buff.error || bit_buff.pos < bit_buff.end) + { /* info_length was wrong */ + my_errno=HA_ERR_WRONG_IN_RECORD; + my_free((gptr) share->decode_trees,MYF(0)); + my_free((gptr) share->decode_tables,MYF(0)); + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} + + + /* Read on huff-code-table from datafile */ + +static void read_huff_table(MI_BIT_BUFF *bit_buff, MI_DECODE_TREE *decode_tree, + uint16 **decode_table, byte **intervall_buff, + uint16 *tmp_buff) +{ + uint min_chr,elements,char_bits,offset_bits,size,intervall_length,table_bits, + next_free_offset; + uint16 *ptr,*end; + + LINT_INIT(ptr); + if (!get_bits(bit_buff,1)) + { + min_chr=get_bits(bit_buff,8); + elements=get_bits(bit_buff,9); + char_bits=get_bits(bit_buff,5); + offset_bits=get_bits(bit_buff,5); + intervall_length=0; + ptr=tmp_buff; + } + else + { + min_chr=0; + elements=get_bits(bit_buff,15); + intervall_length=get_bits(bit_buff,16); + char_bits=get_bits(bit_buff,5); + offset_bits=get_bits(bit_buff,5); + decode_tree->quick_table_bits=0; + ptr= *decode_table; + } + size=elements*2-2; + + for (end=ptr+size ; ptr < end ; ptr++) + { + if (get_bit(bit_buff)) + *ptr= (uint16) get_bits(bit_buff,offset_bits); + else + *ptr= (uint16) (IS_CHAR + (get_bits(bit_buff,char_bits) + min_chr)); + } + skipp_to_next_byte(bit_buff); + + decode_tree->table= *decode_table; + decode_tree->intervalls= *intervall_buff; + if (! intervall_length) + { + table_bits=find_longest_bitstream(tmp_buff); + if (table_bits > myisam_quick_table_bits) + table_bits=myisam_quick_table_bits; + next_free_offset= (1 << table_bits); + make_quick_table(*decode_table,tmp_buff,&next_free_offset,0,table_bits, + table_bits); + (*decode_table)+= next_free_offset; + decode_tree->quick_table_bits=table_bits; + } + else + { + (*decode_table)=end; + bit_buff->pos-= bit_buff->bits/8; + memcpy(*intervall_buff,bit_buff->pos,(size_t) intervall_length); + (*intervall_buff)+=intervall_length; + bit_buff->pos+=intervall_length; + bit_buff->bits=0; + } + return; +} + + +static void make_quick_table(uint16 *to_table, uint16 *decode_table, + uint *next_free_offset, uint value, uint bits, + uint max_bits) +{ + if (!bits--) + { + to_table[value]= (uint16) *next_free_offset; + *next_free_offset=copy_decode_table(to_table, *next_free_offset, + decode_table); + return; + } + if (!(*decode_table & IS_CHAR)) + { + make_quick_table(to_table,decode_table+ *decode_table, + next_free_offset,value,bits,max_bits); + } + else + fill_quick_table(to_table+value,bits,max_bits,(uint) *decode_table); + decode_table++; + value|= (1 << bits); + if (!(*decode_table & IS_CHAR)) + { + make_quick_table(to_table,decode_table+ *decode_table, + next_free_offset,value,bits,max_bits); + } + else + fill_quick_table(to_table+value,bits,max_bits,(uint) *decode_table); + return; +} + + +static void fill_quick_table(uint16 *table, uint bits, uint max_bits, + uint value) +{ + uint16 *end; + value|=(max_bits-bits) << 8; + for (end=table+ (1 << bits) ; + table < end ; + *table++ = (uint16) value | IS_CHAR) ; +} + + +static uint copy_decode_table(uint16 *to_pos, uint offset, + uint16 *decode_table) +{ + uint prev_offset; + prev_offset= offset; + + if (!(*decode_table & IS_CHAR)) + { + to_pos[offset]=2; + offset=copy_decode_table(to_pos,offset+2,decode_table+ *decode_table); + } + else + { + to_pos[offset]= *decode_table; + offset+=2; + } + decode_table++; + + if (!(*decode_table & IS_CHAR)) + { + to_pos[prev_offset+1]=(uint16) (offset-prev_offset-1); + offset=copy_decode_table(to_pos,offset,decode_table+ *decode_table); + } + else + to_pos[prev_offset+1]= *decode_table; + return offset; +} + + +static uint find_longest_bitstream(uint16 *table) +{ + uint length=1,length2; + if (!(*table & IS_CHAR)) + length=find_longest_bitstream(table+ *table)+1; + table++; + if (!(*table & IS_CHAR)) + { + length2=find_longest_bitstream(table+ *table)+1; + length=max(length,length2); + } + return length; +} + + + /* Read record from datafile */ + /* Returns length of packed record, -1 if error */ + +int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, byte *buf) +{ + MI_BLOCK_INFO block_info; + File file; + DBUG_ENTER("mi_read_pack_record"); + + if (filepos == HA_OFFSET_ERROR) + DBUG_RETURN(-1); /* _search() didn't find record */ + + file=info->dfile; + if (_mi_pack_get_block_info(info, &block_info, file, filepos, + info->rec_buff)) + goto err; + if (my_read(file,(byte*) info->rec_buff + block_info.offset , + block_info.rec_len - block_info.offset, MYF(MY_NABP))) + goto panic; + info->update|= HA_STATE_AKTIV; + DBUG_RETURN(_mi_pack_rec_unpack(info,buf,info->rec_buff,block_info.rec_len)); +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; +err: + DBUG_RETURN(-1); +} + + + +int _mi_pack_rec_unpack(register MI_INFO *info, register byte *to, byte *from, + ulong reclength) +{ + byte *end_field; + reg3 MI_COLUMNDEF *end; + MI_COLUMNDEF *current_field; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_pack_rec_unpack"); + + init_bit_buffer(&info->bit_buff, (uchar*) from,reclength); + + for (current_field=share->rec, end=current_field+share->base.fields ; + current_field < end ; + current_field++,to=end_field) + { + end_field=to+current_field->length; + (*current_field->unpack)(current_field,&info->bit_buff,(uchar*) to, + (uchar*) end_field); + } + if (! info->bit_buff.error && + info->bit_buff.pos - info->bit_buff.bits/8 == info->bit_buff.end) + DBUG_RETURN(0); + info->update&= ~HA_STATE_AKTIV; + DBUG_RETURN(my_errno=HA_ERR_WRONG_IN_RECORD); +} /* _mi_pack_rec_unpack */ + + + /* Return function to unpack field */ + +static void (*get_unpack_function(MI_COLUMNDEF *rec)) +(MI_COLUMNDEF *, MI_BIT_BUFF *, uchar *, uchar *) +{ + switch (rec->base_type) { + case FIELD_SKIPP_ZERO: + if (rec->pack_type & PACK_TYPE_ZERO_FILL) + return &uf_zerofill_skipp_zero; + return &uf_skipp_zero; + case FIELD_NORMAL: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + return &uf_space_normal; + if (rec->pack_type & PACK_TYPE_ZERO_FILL) + return &uf_zerofill_normal; + return &decode_bytes; + case FIELD_SKIPP_ENDSPACE: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + { + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_space_endspace_selected; + return &uf_space_endspace; + } + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_endspace_selected; + return &uf_endspace; + case FIELD_SKIPP_PRESPACE: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + { + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_space_prespace_selected; + return &uf_space_prespace; + } + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_prespace_selected; + return &uf_prespace; + case FIELD_CONSTANT: + return &uf_constant; + case FIELD_INTERVALL: + return &uf_intervall; + case FIELD_ZERO: + case FIELD_CHECK: + return &uf_zero; + case FIELD_BLOB: + return &uf_blob; + case FIELD_VARCHAR: + return &uf_varchar; + case FIELD_LAST: + default: + return 0; /* This should never happend */ + } +} + + /* De different functions to unpack a field */ + +static void uf_zerofill_skipp_zero(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + if (get_bit(bit_buff)) + bzero((char*) to,(uint) (end-to)); + else + { + end-=rec->space_length_bits; + decode_bytes(rec,bit_buff,to,end); + bzero((char*) end,rec->space_length_bits); + } +} + +static void uf_skipp_zero(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + if (get_bit(bit_buff)) + bzero((char*) to,(uint) (end-to)); + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_normal(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + if (get_bit(bit_buff)) + bfill((byte*) to,(end-to),' '); + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_endspace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((byte*) to,(end-to),' '); + else + { + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((byte*) end-spaces,spaces,' '); + } + else + decode_bytes(rec,bit_buff,to,end); + } +} + +static void uf_endspace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((byte*) end-spaces,spaces,' '); + } + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_endspace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((byte*) to,(end-to),' '); + else + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((byte*) end-spaces,spaces,' '); + } +} + +static void uf_endspace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((byte*) end-spaces,spaces,' '); +} + +static void uf_space_prespace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((byte*) to,(end-to),' '); + else + { + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((byte*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } + else + decode_bytes(rec,bit_buff,to,end); + } +} + + +static void uf_prespace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((byte*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } + else + decode_bytes(rec,bit_buff,to,end); +} + + +static void uf_space_prespace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((byte*) to,(end-to),' '); + else + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((byte*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } +} + +static void uf_prespace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((byte*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); +} + +static void uf_zerofill_normal(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + end-=rec->space_length_bits; + decode_bytes(rec,bit_buff,(uchar*) to,end); + bzero((char*) end,rec->space_length_bits); +} + +static void uf_constant(MI_COLUMNDEF *rec, + MI_BIT_BUFF *bit_buff __attribute__((unused)), + uchar *to, + uchar *end) +{ + memcpy(to,rec->huff_tree->intervalls,(size_t) (end-to)); +} + +static void uf_intervall(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + reg1 uint field_length=(uint) (end-to); + memcpy(to,rec->huff_tree->intervalls+field_length*decode_pos(bit_buff, + rec->huff_tree), + (size_t) field_length); +} + + +/*ARGSUSED*/ +static void uf_zero(MI_COLUMNDEF *rec __attribute__((unused)), + MI_BIT_BUFF *bit_buff __attribute__((unused)), + uchar *to, uchar *end) +{ + bzero((char*) to,(uint) (end-to)); +} + +static void uf_blob(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + if (get_bit(bit_buff)) + bzero((byte*) to,(end-to)); + else + { + ulong length=get_bits(bit_buff,rec->space_length_bits); + uint pack_length=(uint) (end-to)-mi_portable_sizeof_char_ptr; + decode_bytes(rec,bit_buff,bit_buff->blob_pos,bit_buff->blob_pos+length); + _my_store_blob_length((byte*) to,pack_length,length); + memcpy_fixed((char*) to+pack_length,(char*) &bit_buff->blob_pos, + sizeof(char*)); + bit_buff->blob_pos+=length; + } +} + +static void uf_varchar(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end __attribute__((unused))) +{ + if (get_bit(bit_buff)) + to[0]=to[1]=0; /* Zero lengths */ + else + { + ulong length=get_bits(bit_buff,rec->space_length_bits); + int2store(to,length); + decode_bytes(rec,bit_buff,to+2,to+2+length); + } +} + + /* Functions to decode of buffer of bits */ + +#if BITS_SAVED == 64 + +static void decode_bytes(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff,uchar *to, + uchar *end) +{ + reg1 uint bits,low_byte; + reg3 uint16 *pos; + reg4 uint table_bits,table_and; + MI_DECODE_TREE *decode_tree; + + decode_tree=rec->decode_tree; + bits=bit_buff->bits; /* Save in reg for quicker access */ + table_bits=decode_tree->quick_table_bits; + table_and= (1 << table_bits)-1; + + do + { + if (bits <= 32) + { + if (bit_buff->pos > bit_buff->end+4) + return; /* Can't be right */ + bit_buff->current_byte= (bit_buff->current_byte << 32) + + ((((uint) bit_buff->pos[3])) + + (((uint) bit_buff->pos[2]) << 8) + + (((uint) bit_buff->pos[1]) << 16) + + (((uint) bit_buff->pos[0]) << 24)); + bit_buff->pos+=4; + bits+=32; + } + /* First use info in quick_table */ + low_byte=(uint) (bit_buff->current_byte >> (bits - table_bits)) & table_and; + low_byte=decode_tree->table[low_byte]; + if (low_byte & IS_CHAR) + { + *to++ = (low_byte & 255); /* Found char in quick table */ + bits-= ((low_byte >> 8) & 31); /* Remove bits used */ + } + else + { /* Map through rest of decode-table */ + pos=decode_tree->table+low_byte; + bits-=table_bits; + for (;;) + { + low_byte=(uint) (bit_buff->current_byte >> (bits-8)); + decode_bytes_test_bit(0); + decode_bytes_test_bit(1); + decode_bytes_test_bit(2); + decode_bytes_test_bit(3); + decode_bytes_test_bit(4); + decode_bytes_test_bit(5); + decode_bytes_test_bit(6); + decode_bytes_test_bit(7); + bits-=8; + } + *to++ = *pos; + } + } while (to != end); + + bit_buff->bits=bits; + return; +} + +#else + +static void decode_bytes(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, uchar *end) +{ + reg1 uint bits,low_byte; + reg3 uint16 *pos; + reg4 uint table_bits,table_and; + MI_DECODE_TREE *decode_tree; + + decode_tree=rec->huff_tree; + bits=bit_buff->bits; /* Save in reg for quicker access */ + table_bits=decode_tree->quick_table_bits; + table_and= (1 << table_bits)-1; + + do + { + if (bits < table_bits) + { + if (bit_buff->pos > bit_buff->end+1) + return; /* Can't be right */ +#if BITS_SAVED == 32 + bit_buff->current_byte= (bit_buff->current_byte << 24) + + (((uint) ((uchar) bit_buff->pos[2]))) + + (((uint) ((uchar) bit_buff->pos[1])) << 8) + + (((uint) ((uchar) bit_buff->pos[0])) << 16); + bit_buff->pos+=3; + bits+=24; +#else + if (bits) /* We must have at leasts 9 bits */ + { + bit_buff->current_byte= (bit_buff->current_byte << 8) + + (uint) ((uchar) bit_buff->pos[0]); + bit_buff->pos++; + bits+=8; + } + else + { + bit_buff->current_byte= ((uint) ((uchar) bit_buff->pos[0]) << 8) + + ((uint) ((uchar) bit_buff->pos[1])); + bit_buff->pos+=2; + bits+=16; + } +#endif + } + /* First use info in quick_table */ + low_byte=(bit_buff->current_byte >> (bits - table_bits)) & table_and; + low_byte=decode_tree->table[low_byte]; + if (low_byte & IS_CHAR) + { + *to++ = (low_byte & 255); /* Found char in quick table */ + bits-= ((low_byte >> 8) & 31); /* Remove bits used */ + } + else + { /* Map through rest of decode-table */ + pos=decode_tree->table+low_byte; + bits-=table_bits; + for (;;) + { + if (bits < 8) + { /* We don't need to check end */ +#if BITS_SAVED == 32 + bit_buff->current_byte= (bit_buff->current_byte << 24) + + (((uint) ((uchar) bit_buff->pos[2]))) + + (((uint) ((uchar) bit_buff->pos[1])) << 8) + + (((uint) ((uchar) bit_buff->pos[0])) << 16); + bit_buff->pos+=3; + bits+=24; +#else + bit_buff->current_byte= (bit_buff->current_byte << 8) + + (uint) ((uchar) bit_buff->pos[0]); + bit_buff->pos+=1; + bits+=8; +#endif + } + low_byte=(uint) (bit_buff->current_byte >> (bits-8)); + decode_bytes_test_bit(0); + decode_bytes_test_bit(1); + decode_bytes_test_bit(2); + decode_bytes_test_bit(3); + decode_bytes_test_bit(4); + decode_bytes_test_bit(5); + decode_bytes_test_bit(6); + decode_bytes_test_bit(7); + bits-=8; + } + *to++ = (uchar) *pos; + } + } while (to != end); + + bit_buff->bits=bits; + return; +} +#endif /* BIT_SAVED == 64 */ + + +static uint decode_pos(MI_BIT_BUFF *bit_buff, MI_DECODE_TREE *decode_tree) +{ + uint16 *pos=decode_tree->table; + for (;;) + { + if (get_bit(bit_buff)) + pos++; + if (*pos & IS_CHAR) + return (uint) (*pos & ~IS_CHAR); + pos+= *pos; + } +} + + +int _mi_read_rnd_pack_record(MI_INFO *info, byte *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks) +{ + uint b_type; + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_pack_record"); + + if (filepos >= info->state->data_file_length) + { + my_errno= HA_ERR_END_OF_FILE; + goto err; + } + + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(byte*) block_info.header,filepos, + share->pack.ref_length, skip_deleted_blocks)) + goto err; + b_type=_mi_pack_get_block_info(info,&block_info,-1, filepos, NullS); + } + else + b_type=_mi_pack_get_block_info(info,&block_info,info->dfile,filepos, + info->rec_buff); + if (b_type) + goto err; +#ifndef DBUG_OFF + if (block_info.rec_len > share->max_pack_length) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } +#endif + + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(byte*) info->rec_buff, + block_info.filepos, block_info.rec_len, + skip_deleted_blocks)) + goto err; + } + else + { + if (my_read(info->dfile,(byte*) info->rec_buff + block_info.offset, + block_info.rec_len-block_info.offset, + MYF(MY_NABP))) + goto err; + } + info->packed_length=block_info.rec_len; + info->lastpos=filepos; + info->nextpos=block_info.filepos+block_info.rec_len; + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + + DBUG_RETURN (_mi_pack_rec_unpack(info,buf,info->rec_buff, + block_info.rec_len)); + err: + DBUG_RETURN(my_errno); +} + + + /* Read and process header from a huff-record-file */ + +uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BLOCK_INFO *info, File file, + my_off_t filepos, char *rec_buff) +{ + uchar *header=info->header; + uint head_length,ref_length; + LINT_INIT(ref_length); + + if (file >= 0) + { + ref_length=myisam->s->pack.ref_length; + /* + We can't use my_pread() here because mi_rad_pack_record assumes + position is ok + */ + VOID(my_seek(file,filepos,MY_SEEK_SET,MYF(0))); + if (my_read(file,(char*) header,ref_length,MYF(MY_NABP))) + return BLOCK_FATAL_ERROR; + DBUG_DUMP("header",(byte*) header,ref_length); + } + if (header[0] < 254) + { + info->rec_len=header[0]; + head_length=1; + } + else if (header[0] == 254) + { + info->rec_len=uint2korr(header+1); + head_length=3; + } + else + { + info->rec_len=uint3korr(header+1); + head_length=4; + } + if (myisam->s->base.blobs) + { + if (header[head_length] < 254) + { + info->blob_len=header[head_length]; + head_length++; + } + else if (header[head_length] == 254) + { + info->blob_len=uint2korr(header+head_length+1); + head_length+=3; + } + else + { + info->blob_len=uint3korr(header+head_length+1); + head_length+=4; + } + if (!(mi_fix_rec_buff_for_blob(myisam,info->rec_len + info->blob_len))) + return BLOCK_FATAL_ERROR; /* not enough memory */ + myisam->bit_buff.blob_pos=(uchar*) myisam->rec_buff+info->rec_len; + myisam->blob_length=info->blob_len; + } + info->filepos=filepos+head_length; + if (file > 0) + { + info->offset=min(info->rec_len, ref_length - head_length); + memcpy(rec_buff, header+head_length, info->offset); + } + return 0; +} + + + /* rutines for bit buffer */ + /* Note buffer must be 6 byte bigger than longest row */ + +static void init_bit_buffer(MI_BIT_BUFF *bit_buff, uchar *buffer, uint length) +{ + bit_buff->pos=buffer; + bit_buff->end=buffer+length; + bit_buff->bits=bit_buff->error=0; + bit_buff->current_byte=0; /* Avoid purify errors */ +} + +static uint fill_and_get_bits(MI_BIT_BUFF *bit_buff, uint count) +{ + uint tmp; + count-=bit_buff->bits; + tmp=(bit_buff->current_byte & mask[bit_buff->bits]) << count; + fill_buffer(bit_buff); + bit_buff->bits=BITS_SAVED - count; + return tmp+(bit_buff->current_byte >> (BITS_SAVED - count)); +} + + /* Fill in empty bit_buff->current_byte from buffer */ + /* Sets bit_buff->error if buffer is exhausted */ + +static void fill_buffer(MI_BIT_BUFF *bit_buff) +{ + if (bit_buff->pos >= bit_buff->end) + { + bit_buff->error= 1; + bit_buff->current_byte=0; + return; + } +#if BITS_SAVED == 64 + bit_buff->current_byte= ((((uint) ((uchar) bit_buff->pos[7]))) + + (((uint) ((uchar) bit_buff->pos[6])) << 8) + + (((uint) ((uchar) bit_buff->pos[5])) << 16) + + (((uint) ((uchar) bit_buff->pos[4])) << 24) + + ((ulonglong) + ((((uint) ((uchar) bit_buff->pos[3]))) + + (((uint) ((uchar) bit_buff->pos[2])) << 8) + + (((uint) ((uchar) bit_buff->pos[1])) << 16) + + (((uint) ((uchar) bit_buff->pos[0])) << 24)) << 32)); + bit_buff->pos+=8; +#else +#if BITS_SAVED == 32 + bit_buff->current_byte= (((uint) ((uchar) bit_buff->pos[3])) + + (((uint) ((uchar) bit_buff->pos[2])) << 8) + + (((uint) ((uchar) bit_buff->pos[1])) << 16) + + (((uint) ((uchar) bit_buff->pos[0])) << 24)); + bit_buff->pos+=4; +#else + bit_buff->current_byte= (uint) (((uint) ((uchar) bit_buff->pos[1]))+ + (((uint) ((uchar) bit_buff->pos[0])) << 8)); + bit_buff->pos+=2; +#endif +#endif +} + + /* Get number of bits neaded to represent value */ + +static uint max_bit(register uint value) +{ + reg2 uint power=1; + + while ((value>>=1)) + power++; + return (power); +} + + +/***************************************************************************** + Some redefined functions to handle files when we are using memmap +*****************************************************************************/ + +#ifdef HAVE_MMAP + +#include <sys/mman.h> + +static int _mi_read_mempack_record(MI_INFO *info,my_off_t filepos,byte *buf); +static int _mi_read_rnd_mempack_record(MI_INFO*, byte *,my_off_t, my_bool); + +#ifndef MAP_NORESERVE +#define MAP_NORESERVE 0 /* For irix */ +#endif +#ifndef MAP_FAILED +#define MAP_FAILED -1 +#endif + +my_bool _mi_memmap_file(MI_INFO *info) +{ + byte *file_map; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_memmap_file"); + + if (!info->s->file_map) + { + if (my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)) < + share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN) + { + DBUG_PRINT("warning",("File isn't extended for memmap")); + DBUG_RETURN(0); + } + file_map=(byte*) + mmap(0,share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN,PROT_READ, + MAP_SHARED | MAP_NORESERVE,info->dfile,0L); + if (file_map == (byte*) MAP_FAILED) + { + DBUG_PRINT("warning",("mmap failed: errno: %d",errno)); + my_errno=errno; + DBUG_RETURN(0); + } + info->s->file_map=file_map; + } + info->opt_flag|= MEMMAP_USED; + info->read_record=share->read_record=_mi_read_mempack_record; + share->read_rnd=_mi_read_rnd_mempack_record; + DBUG_RETURN(1); +} + + +void _mi_unmap_file(MI_INFO *info) +{ + VOID(munmap((caddr_t) info->s->file_map, + (size_t) info->s->state.state.data_file_length+ + MEMMAP_EXTRA_MARGIN)); +} + + +static uchar *_mi_mempack_get_block_info(MI_INFO *myisam,MI_BLOCK_INFO *info, + uchar *header) +{ + if (header[0] < 254) + info->rec_len= *header++; + else if (header[0] == 254) + { + info->rec_len=uint2korr(header+1); + header+=3; + } + else + { + info->rec_len=uint3korr(header+1); + header+=4; + } + if (myisam->s->base.blobs) + { + if (header[0] < 254) + { + info->blob_len= *header++; + } + else if (header[0] == 254) + { + info->blob_len=uint2korr(header+1); + header+=3; + } + else + { + info->blob_len=uint3korr(header+1); + header+=4; + } + /* mi_fix_rec_buff_for_blob sets my_errno on error */ + if (!(mi_fix_rec_buff_for_blob(myisam,info->blob_len))) + return 0; /* not enough memory */ + myisam->bit_buff.blob_pos=(uchar*) myisam->rec_buff; + } + return header; +} + + +static int _mi_read_mempack_record(MI_INFO *info, my_off_t filepos, byte *buf) +{ + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + byte *pos; + DBUG_ENTER("mi_read_mempack_record"); + + if (filepos == HA_OFFSET_ERROR) + DBUG_RETURN(-1); /* _search() didn't find record */ + + if (!(pos= (byte*) _mi_mempack_get_block_info(info,&block_info, + (uchar*) share->file_map+ + filepos))) + DBUG_RETURN(-1); + DBUG_RETURN(_mi_pack_rec_unpack(info, buf, pos, block_info.rec_len)); +} + + +/*ARGSUSED*/ +static int _mi_read_rnd_mempack_record(MI_INFO *info, byte *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks + __attribute__((unused))) +{ + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + byte *pos,*start; + DBUG_ENTER("_mi_read_rnd_mempack_record"); + + if (filepos >= share->state.state.data_file_length) + { + my_errno=HA_ERR_END_OF_FILE; + goto err; + } + if (!(pos= (byte*) _mi_mempack_get_block_info(info,&block_info, + (uchar*) + (start=share->file_map+ + filepos)))) + goto err; +#ifndef DBUG_OFF + if (block_info.rec_len > info->s->max_pack_length) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } +#endif + info->packed_length=block_info.rec_len; + info->lastpos=filepos; + info->nextpos=filepos+(uint) (pos-start)+block_info.rec_len; + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + + DBUG_RETURN (_mi_pack_rec_unpack(info,buf,pos, block_info.rec_len)); + err: + DBUG_RETURN(my_errno); +} + +#endif /* HAVE_MMAP */ + + /* Save length of row */ + +uint save_pack_length(byte *block_buff,ulong length) +{ + if (length < 254) + { + *(uchar*) block_buff= (uchar) length; + return 1; + } + if (length <= 65535) + { + *(uchar*) block_buff=254; + int2store(block_buff+1,(uint) length); + return 3; + } + *(uchar*) block_buff=255; + int3store(block_buff+1,(ulong) length); + return 4; +} diff --git a/myisam/mi_page.c b/myisam/mi_page.c new file mode 100644 index 00000000000..dd7de9ba8b6 --- /dev/null +++ b/myisam/mi_page.c @@ -0,0 +1,144 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Read and write key blocks */ + +#include "myisamdef.h" +#ifdef __WIN__ +#include <errno.h> +#endif + + /* Fetch a key-page in memory */ + +uchar *_mi_fetch_keypage(register MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, int return_buffer) +{ + uchar *tmp; + uint page_size; + tmp=(uchar*) key_cache_read(info->s->kfile,page,(byte*) buff, + (uint) keyinfo->block_length, + (uint) keyinfo->block_length, + return_buffer); + if (tmp == info->buff) + info->buff_used=1; + else if (!tmp) + { + DBUG_PRINT("error",("Got errno: %d from key_cache_read",my_errno)); + info->last_keypage=HA_OFFSET_ERROR; + my_errno=HA_ERR_CRASHED; + return 0; + } + info->last_keypage=page; + page_size=mi_getint(tmp); + if (page_size < 4 || page_size > keyinfo->block_length) + { + DBUG_PRINT("error",("page %lu had wrong page length: %u", + (ulong) page, page_size)); + info->last_keypage = HA_OFFSET_ERROR; + my_errno = HA_ERR_CRASHED; + tmp = 0; + } + return tmp; +} /* _mi_fetch_keypage */ + + + /* Write a key-page on disk */ + +int _mi_write_keypage(register MI_INFO *info, register MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff) +{ + reg3 uint length; +#ifndef FAST /* Safety check */ + if (page < info->s->base.keystart || + page+keyinfo->block_length > info->state->key_file_length || + page & (myisam_block_size-1)) + { + DBUG_PRINT("error",("Trying to write outside key region: %lu", + (long) page)); + my_errno=EINVAL; + return(-1); + } + DBUG_PRINT("page",("write page at: %lu",(long) page,buff)); + DBUG_DUMP("buff",(byte*) buff,mi_getint(buff)); +#endif + + if ((length=keyinfo->block_length) > IO_SIZE*2 && + info->state->key_file_length != page+length) + length= ((mi_getint(buff)+IO_SIZE-1) & (uint) ~(IO_SIZE-1)); +#ifdef HAVE_purify + { + length=mi_getint(buff); + bzero((byte*) buff+length,keyinfo->block_length-length); + length=keyinfo->block_length; + } +#endif + return (key_cache_write(info->s->kfile,page,(byte*) buff,length, + (uint) keyinfo->block_length, + (int) ((info->lock_type != F_UNLCK) || + info->s->delay_key_write))); +} /* mi_write_keypage */ + + + /* Remove page from disk */ + +int _mi_dispose(register MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos) +{ + my_off_t old_link; + char buff[8]; + DBUG_ENTER("_mi_dispose"); + + old_link=info->s->state.key_del[keyinfo->block_size]; + info->s->state.key_del[keyinfo->block_size]=pos; + mi_sizestore(buff,old_link); + DBUG_RETURN(key_cache_write(info->s->kfile,pos,buff, + sizeof(buff), + (uint) keyinfo->block_length, + (int) (info->lock_type != F_UNLCK))); +} /* _mi_dispose */ + + + /* Make new page on disk */ + +my_off_t _mi_new(register MI_INFO *info, MI_KEYDEF *keyinfo) +{ + my_off_t pos; + char buff[8]; + DBUG_ENTER("_mi_new"); + + if ((pos=info->s->state.key_del[keyinfo->block_size]) == HA_OFFSET_ERROR) + { + if (info->state->key_file_length >= + info->s->base.max_key_file_length - keyinfo->block_length) + { + my_errno=HA_ERR_INDEX_FILE_FULL; + DBUG_RETURN(HA_OFFSET_ERROR); + } + pos=info->state->key_file_length; + info->state->key_file_length+= keyinfo->block_length; + } + else + { + if (!key_cache_read(info->s->kfile,pos, + buff, + (uint) sizeof(buff), + (uint) keyinfo->block_length,0)) + pos= HA_OFFSET_ERROR; + else + info->s->state.key_del[keyinfo->block_size]=mi_sizekorr(buff); + } + DBUG_PRINT("exit",("Pos: %d",pos)); + DBUG_RETURN(pos); +} /* _mi_new */ diff --git a/myisam/mi_panic.c b/myisam/mi_panic.c new file mode 100644 index 00000000000..ae149df3ffe --- /dev/null +++ b/myisam/mi_panic.c @@ -0,0 +1,111 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* if flag == HA_PANIC_CLOSE then all misam files are closed */ + /* if flag == HA_PANIC_WRITE then all misam files are unlocked and + all changed data in single user misam is written to file */ + /* if flag == HA_PANIC_READ then all misam files that was locked when + mi_panic(HA_PANIC_WRITE) was done is locked. A mi_readinfo() is + done for all single user files to get changes in database */ + + +int mi_panic(enum ha_panic_function flag) +{ + int error=0; + LIST *list_element,*next_open; + MI_INFO *info; + DBUG_ENTER("mi_panic"); + + pthread_mutex_lock(&THR_LOCK_myisam); + for (list_element=myisam_open_list ; list_element ; list_element=next_open) + { + next_open=list_element->next; /* Save if close */ + info=(MI_INFO*) list_element->data; + switch (flag) { + case HA_PANIC_CLOSE: + pthread_mutex_unlock(&THR_LOCK_myisam); /* Not exactly right... */ + if (mi_close(info)) + error=my_errno; + pthread_mutex_lock(&THR_LOCK_myisam); + break; + case HA_PANIC_WRITE: /* Do this to free databases */ +#ifdef CANT_OPEN_FILES_TWICE + if (info->s->options & HA_OPTION_READ_ONLY_DATA) + break; +#endif + if (flush_key_blocks(info->s->kfile,FLUSH_RELEASE)) + error=my_errno; + if (info->opt_flag & WRITE_CACHE_USED) + if (flush_io_cache(&info->rec_cache)) + error=my_errno; + if (info->opt_flag & READ_CACHE_USED) + { + if (flush_io_cache(&info->rec_cache)) + error=my_errno; + reinit_io_cache(&info->rec_cache,READ_CACHE,0, + (pbool) (info->lock_type != F_UNLCK),1); + } + if (info->lock_type != F_UNLCK && ! info->was_locked) + { + info->was_locked=info->lock_type; + if (mi_lock_database(info,F_UNLCK)) + error=my_errno; + } +#ifdef CANT_OPEN_FILES_TWICE + if (info->s->kfile >= 0 && my_close(info->s->kfile,MYF(0))) + error = my_errno; + if (info->dfile >= 0 && my_close(info->dfile,MYF(0))) + error = my_errno; + info->s->kfile=info->dfile= -1; /* Files aren't open anymore */ + break; +#endif + case HA_PANIC_READ: /* Restore to before WRITE */ +#ifdef CANT_OPEN_FILES_TWICE + { /* Open closed files */ + char name_buff[FN_REFLEN]; + if (info->s->kfile < 0) + if ((info->s->kfile= my_open(fn_format(name_buff,info->filename,"", + N_NAME_IEXT,4),info->mode, + MYF(MY_WME))) < 0) + error = my_errno; + if (info->dfile < 0) + { + if ((info->dfile= my_open(fn_format(name_buff,info->filename,"", + N_NAME_DEXT,4),info->mode, + MYF(MY_WME))) < 0) + error = my_errno; + info->rec_cache.file=info->dfile; + } + } +#endif + if (info->was_locked) + { + if (mi_lock_database(info, info->was_locked)) + error=my_errno; + info->was_locked=0; + } + break; + } + } + if (flag == HA_PANIC_CLOSE) + VOID(mi_log(0)); /* Close log if neaded */ + pthread_mutex_unlock(&THR_LOCK_myisam); + if (!error) + DBUG_RETURN(0); + DBUG_RETURN(my_errno=error); +} /* mi_panic */ diff --git a/myisam/mi_range.c b/myisam/mi_range.c new file mode 100644 index 00000000000..038f9abc3a6 --- /dev/null +++ b/myisam/mi_range.c @@ -0,0 +1,199 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Gives a approximated number of how many records there is between two keys. + Used when optimizing querries. + */ + +#include "myisamdef.h" + +static ha_rows _mi_record_pos(MI_INFO *info,const byte *key,uint key_len, + enum ha_rkey_function search_flag); +static double _mi_search_pos(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uint key_len,uint nextflag,my_off_t pos); +static uint _mi_keynr(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *page, + uchar *keypos,uint *ret_max_key); + + + /* If start_key = 0 assume read from start */ + /* If end_key = 0 assume read to end */ + /* Returns HA_POS_ERROR on error */ + +ha_rows mi_records_in_range(MI_INFO *info, int inx, const byte *start_key, + uint start_key_len, + enum ha_rkey_function start_search_flag, + const byte *end_key, uint end_key_len, + enum ha_rkey_function end_search_flag) +{ + ha_rows start_pos,end_pos; + DBUG_ENTER("mi_records_in_range"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(HA_POS_ERROR); + + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(HA_POS_ERROR); + info->update&= (HA_STATE_CHANGED+HA_STATE_ROW_CHANGED); + if (info->s->concurrent_insert) + rw_rdlock(&info->s->key_root_lock[inx]); + start_pos= (start_key ? + _mi_record_pos(info,start_key,start_key_len,start_search_flag) : + (ha_rows) 0); + end_pos= (end_key ? + _mi_record_pos(info,end_key,end_key_len,end_search_flag) : + info->state->records+ (ha_rows) 1); + if (info->s->concurrent_insert) + rw_unlock(&info->s->key_root_lock[inx]); + VOID(_mi_writeinfo(info,0)); + if (start_pos == HA_POS_ERROR || end_pos == HA_POS_ERROR) + DBUG_RETURN(HA_POS_ERROR); + DBUG_PRINT("info",("records: %ld",(ulong) (end_pos-start_pos))); + DBUG_RETURN(end_pos < start_pos ? (ha_rows) 0 : + (end_pos == start_pos ? (ha_rows) 1 : end_pos-start_pos)); +} + + + /* Find relative position (in records) for key in index-tree */ + +static ha_rows _mi_record_pos(MI_INFO *info, const byte *key, uint key_len, + enum ha_rkey_function search_flag) +{ + uint inx=(uint) info->lastinx; + MI_KEYDEF *keyinfo=info->s->keyinfo+inx; + uchar *key_buff; + double pos; + + DBUG_ENTER("_mi_record_pos"); + DBUG_PRINT("enter",("search_flag: %d",search_flag)); + + if (key_len == 0) + key_len=USE_WHOLE_KEY; + key_buff=info->lastkey+info->s->base.max_key_length; + key_len=_mi_pack_key(info,inx,key_buff,(uchar*) key,key_len); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg, + (uchar*) key_buff,key_len);); + pos=_mi_search_pos(info,keyinfo,key_buff,key_len, + myisam_read_vec[search_flag] | SEARCH_SAVE_BUFF, + info->s->state.key_root[inx]); + if (pos >= 0.0) + { + DBUG_PRINT("exit",("pos: %ld",(ulong) (pos*info->state->records))); + DBUG_RETURN((ulong) (pos*info->state->records+0.5)); + } + DBUG_RETURN(HA_POS_ERROR); +} + + + /* This is a modified version of _mi_search */ + /* Returns offset for key in indextable (decimal 0.0 <= x <= 1.0) */ + +static double _mi_search_pos(register MI_INFO *info, + register MI_KEYDEF *keyinfo, + uchar *key, uint key_len, uint nextflag, + register my_off_t pos) +{ + int flag; + uint nod_flag,keynr,max_keynr; + my_bool after_key; + uchar *keypos,*buff; + double offset; + DBUG_ENTER("_mi_search_pos"); + + if (pos == HA_OFFSET_ERROR) + DBUG_RETURN(0.5); + + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,info->buff,1))) + goto err; + flag=(*keyinfo->bin_search)(info,keyinfo,buff,key,key_len,nextflag, + &keypos,info->lastkey, &after_key); + nod_flag=mi_test_if_nod(buff); + keynr=_mi_keynr(info,keyinfo,buff,keypos,&max_keynr); + + if (flag) + { + if (flag == MI_FOUND_WRONG_KEY) + DBUG_RETURN(-1); /* error */ + /* + ** Didn't found match. keypos points at next (bigger) key + * Try to find a smaller, better matching key. + ** Matches keynr + [0-1] + */ + if (flag > 0 && ! nod_flag) + offset= 1.0; + else if ((offset=_mi_search_pos(info,keyinfo,key,key_len,nextflag, + _mi_kpos(nod_flag,keypos))) < 0) + DBUG_RETURN(offset); + } + else + { + /* + ** Found match. Keypos points at the start of the found key + ** Matches keynr+1 + */ + offset=1.0; /* Matches keynr+1 */ + if (nextflag & SEARCH_FIND && + ((keyinfo->flag & (HA_NOSAME | HA_NULL_PART)) != HA_NOSAME || + key_len) && nod_flag) + { + /* + ** There may be identical keys in the tree. Try to match on of those. + ** Matches keynr + [0-1] + */ + if ((offset=_mi_search_pos(info,keyinfo,key,key_len,SEARCH_FIND, + _mi_kpos(nod_flag,keypos))) < 0) + DBUG_RETURN(offset); /* Read error */ + } + } + DBUG_PRINT("info",("keynr: %d offset: %g max_keynr: %d nod: %d flag: %d", + keynr,offset,max_keynr,nod_flag,flag)); + DBUG_RETURN((keynr+offset)/(max_keynr+1)); +err: + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1.0); +} + + + /* Get keynummer of current key and max number of keys in nod */ + +static uint _mi_keynr(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, uchar *keypos, uint *ret_max_key) +{ + uint nod_flag,keynr,max_key; + uchar t_buff[MI_MAX_KEY_BUFF],*end; + + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + + if (!(keyinfo->flag & (HA_VAR_LENGTH_KEY| HA_BINARY_PACK_KEY))) + { + *ret_max_key= (uint) (end-page)/(keyinfo->keylength+nod_flag); + return (uint) (keypos-page)/(keyinfo->keylength+nod_flag); + } + + max_key=keynr=0; + t_buff[0]=0; /* Safety */ + while (page < end) + { + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&page,t_buff)) + return 0; /* Error */ + max_key++; + if (page == keypos) + keynr=max_key; + } + *ret_max_key=max_key; + return(keynr); +} diff --git a/myisam/mi_rename.c b/myisam/mi_rename.c new file mode 100644 index 00000000000..6849e1c3d26 --- /dev/null +++ b/myisam/mi_rename.c @@ -0,0 +1,56 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Rename a table +*/ + +#include "fulltext.h" +#ifdef __WIN__ +#include <errno.h> +#endif + +int mi_rename(const char *old_name, const char *new_name) +{ + char from[FN_REFLEN],to[FN_REFLEN]; +#ifdef USE_RAID + uint raid_type=0,raid_chunks=0; +#endif + DBUG_ENTER("mi_rename"); +#ifdef USE_RAID + { + MI_INFO *info; + if (!(info=mi_open(old_name, O_RDONLY, 0))) + DBUG_RETURN(my_errno); + raid_type = info->s->base.raid_type; + raid_chunks = info->s->base.raid_chunks; + mi_close(info); + } +#endif + + fn_format(from,old_name,"",MI_NAME_IEXT,4); + fn_format(to,new_name,"",MI_NAME_IEXT,4); + if (my_rename(from, to, MYF(MY_WME))) + DBUG_RETURN(my_errno); + fn_format(from,old_name,"",MI_NAME_DEXT,4); + fn_format(to,new_name,"",MI_NAME_DEXT,4); +#ifdef USE_RAID + if (raid_type) + DBUG_RETURN(my_raid_rename(from, to, raid_chunks, MYF(MY_WME)) ? my_errno : + 0); +#endif + DBUG_RETURN(my_rename(from, to,MYF(MY_WME)) ? my_errno : 0); +} diff --git a/myisam/mi_rfirst.c b/myisam/mi_rfirst.c new file mode 100644 index 00000000000..8928c6332c5 --- /dev/null +++ b/myisam/mi_rfirst.c @@ -0,0 +1,27 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* Read first row through a specfic key */ + +int mi_rfirst(MI_INFO *info, byte *buf, int inx) +{ + DBUG_ENTER("mi_rfirst"); + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_PREV_FOUND; + DBUG_RETURN(mi_rnext(info,buf,inx)); +} /* mi_rfirst */ diff --git a/myisam/mi_rkey.c b/myisam/mi_rkey.c new file mode 100644 index 00000000000..0b88a62e7fc --- /dev/null +++ b/myisam/mi_rkey.c @@ -0,0 +1,89 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Read record based on a key */ + +#include "myisamdef.h" + + + /* Read a record using key */ + /* Ordinary search_flag is 0 ; Give error if no record with key */ + +int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len, + enum ha_rkey_function search_flag) +{ + uchar *key_buff; + MYISAM_SHARE *share=info->s; + uint pack_key_length; + DBUG_ENTER("mi_rkey"); + DBUG_PRINT("enter",("base: %lx inx: %d search_flag: %d", + info,inx,search_flag)); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + if (key_len == 0) + key_len=USE_WHOLE_KEY; + key_buff=info->lastkey+info->s->base.max_key_length; + pack_key_length=_mi_pack_key(info,(uint) inx,key_buff,(uchar*) key,key_len); + info->last_rkey_length=pack_key_length; + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,share->keyinfo[inx].seg, + key_buff,pack_key_length);); + + if (_mi_readinfo(info,F_RDLCK,1)) + goto err; + if (share->concurrent_insert) + rw_rdlock(&share->key_root_lock[inx]); + if (!_mi_search(info,info->s->keyinfo+inx,key_buff,pack_key_length, + myisam_read_vec[search_flag],info->s->state.key_root[inx])) + { + while (info->lastpos >= info->state->data_file_length) + { + /* + Skip rows that are inserted by other threads since we got a lock + Note that this can only happen if we are not searching after an + exact key, because the keys are sorted according to position + */ + + if (_mi_search_next(info,info->s->keyinfo+inx,info->lastkey, + info->lastkey_length, + myisam_readnext_vec[search_flag], + info->s->state.key_root[inx])) + break; + } + } + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[inx]); + + if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + + info->lastpos = HA_OFFSET_ERROR; /* Didn't find key */ + + /* Store key for read next */ + memcpy(info->lastkey,key_buff,pack_key_length); + bzero((char*) info->lastkey+pack_key_length,info->s->base.rec_reflength); + info->lastkey_length=pack_key_length+info->s->base.rec_reflength; + + if (search_flag == HA_READ_AFTER_KEY) + info->update|=HA_STATE_NEXT_FOUND; /* Previous gives last row */ +err: + DBUG_RETURN(my_errno); +} /* mi_rkey */ diff --git a/myisam/mi_rlast.c b/myisam/mi_rlast.c new file mode 100644 index 00000000000..c08174e9117 --- /dev/null +++ b/myisam/mi_rlast.c @@ -0,0 +1,27 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* Read last row with the same key as the previous read. */ + +int mi_rlast(MI_INFO *info, byte *buf, int inx) +{ + DBUG_ENTER("mi_rlast"); + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_NEXT_FOUND; + DBUG_RETURN(mi_rprev(info,buf,inx)); +} /* mi_rlast */ diff --git a/myisam/mi_rnext.c b/myisam/mi_rnext.c new file mode 100644 index 00000000000..296aa73793a --- /dev/null +++ b/myisam/mi_rnext.c @@ -0,0 +1,84 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* + Read next row with the same key as previous read + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rnext(MI_INFO *info, byte *buf, int inx) +{ + int error; + uint flag; + DBUG_ENTER("mi_rnext"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + flag=SEARCH_BIGGER; /* Read next */ + if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_PREV_FOUND) + flag=0; /* Read first */ + + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(my_errno); + if (info->s->concurrent_insert) + rw_rdlock(&info->s->key_root_lock[inx]); + if (!flag) + error=_mi_search_first(info,info->s->keyinfo+inx, + info->s->state.key_root[inx]); + else if (_mi_test_if_changed(info) == 0) + error=_mi_search_next(info,info->s->keyinfo+inx,info->lastkey, + info->lastkey_length,flag, + info->s->state.key_root[inx]); + else + error=_mi_search(info,info->s->keyinfo+inx,info->lastkey, + info->lastkey_length,flag, info->s->state.key_root[inx]); + + if (!error) + { + while (info->lastpos >= info->state->data_file_length) + { + /* Skip rows that are inserted by other threads since we got a lock */ + if ((error=_mi_search_next(info,info->s->keyinfo+inx,info->lastkey, + info->lastkey_length, + flag, + info->s->state.key_root[inx]))) + break; + } + } + + if (info->s->concurrent_insert) + rw_unlock(&info->s->key_root_lock[inx]); + /* Don't clear if database-changed */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= HA_STATE_NEXT_FOUND; + + if (error) + { + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_PRINT("error",("Got error: %d, errno: %d",error, my_errno)); + DBUG_RETURN(my_errno); +} /* mi_rnext */ diff --git a/myisam/mi_rnext_same.c b/myisam/mi_rnext_same.c new file mode 100644 index 00000000000..0e172894cdf --- /dev/null +++ b/myisam/mi_rnext_same.c @@ -0,0 +1,79 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* + Read next row with the same key as previous read, but abort if + the key changes. + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rnext_same(MI_INFO *info, byte *buf) +{ + int error; + uint inx,flag,not_used; + MI_KEYDEF *keyinfo; + DBUG_ENTER("mi_rnext_same"); + + if ((int) (inx=info->lastinx) < 0 || info->lastpos == HA_OFFSET_ERROR) + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + keyinfo=info->s->keyinfo+inx; + flag=SEARCH_BIGGER; /* Read next */ + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(my_errno); + + memcpy(info->lastkey2,info->lastkey,info->last_rkey_length); + if (info->s->concurrent_insert) + rw_rdlock(&info->s->key_root_lock[inx]); + for (;;) + { + if ((error=_mi_search_next(info,keyinfo,info->lastkey, + info->lastkey_length,flag, + info->s->state.key_root[inx]))) + break; + if (_mi_key_cmp(keyinfo->seg,info->lastkey2,info->lastkey, + info->last_rkey_length, SEARCH_FIND, ¬_used)) + { + error=1; + my_errno=HA_ERR_END_OF_FILE; + info->lastpos= HA_OFFSET_ERROR; + break; + } + /* Skip rows that are inserted by other threads since we got a lock */ + if (info->lastpos < info->state->data_file_length) + break; + } + if (info->s->concurrent_insert) + rw_unlock(&info->s->key_root_lock[inx]); + /* Don't clear if database-changed */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= HA_STATE_NEXT_FOUND; + + if (error) + { + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_RETURN(my_errno); +} /* mi_rnext */ diff --git a/myisam/mi_rprev.c b/myisam/mi_rprev.c new file mode 100644 index 00000000000..f8f89235549 --- /dev/null +++ b/myisam/mi_rprev.c @@ -0,0 +1,82 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* + Read previous row with the same key as previous read + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rprev(MI_INFO *info, byte *buf, int inx) +{ + int error; + register uint flag; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_rprev"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + flag=SEARCH_SMALLER; /* Read previous */ + if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_NEXT_FOUND) + flag=0; /* Read last */ + + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(my_errno); + if (share->concurrent_insert) + rw_rdlock(&share->key_root_lock[inx]); + if (!flag) + error=_mi_search_last(info, share->keyinfo+inx, + share->state.key_root[inx]); + else if (_mi_test_if_changed(info) == 0) + error=_mi_search_next(info,share->keyinfo+inx,info->lastkey, + info->lastkey_length,flag, + share->state.key_root[inx]); + else + error=_mi_search(info,share->keyinfo+inx,info->lastkey, + info->lastkey_length, flag, share->state.key_root[inx]); + + if (!error) + { + while (info->lastpos > info->state->data_file_length) + { + /* Skip rows that are inserted by other threads since we got a lock */ + if ((error=_mi_search_next(info,share->keyinfo+inx,info->lastkey, + info->lastkey_length, + flag, + share->state.key_root[inx]))) + break; + } + } + + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[inx]); + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= HA_STATE_PREV_FOUND; + if (error) + { + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_RETURN(my_errno); +} /* mi_rprev */ diff --git a/myisam/mi_rrnd.c b/myisam/mi_rrnd.c new file mode 100644 index 00000000000..11fa2af59bd --- /dev/null +++ b/myisam/mi_rrnd.c @@ -0,0 +1,57 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Read a record with random-access. The position to the record must + get by MI_INFO. The next record can be read with pos= MI_POS_ERROR */ + + +#include "myisamdef.h" + +/* + Read a row based on position. + If filepos= HA_OFFSET_ERROR then read next row + Return values + Returns one of following values: + 0 = Ok. + HA_ERR_RECORD_DELETED = Record is deleted. + HA_ERR_END_OF_FILE = EOF. +*/ + +int mi_rrnd(MI_INFO *info, byte *buf, register my_off_t filepos) +{ + my_bool skipp_deleted_blocks; + DBUG_ENTER("mi_rrnd"); + + skipp_deleted_blocks=0; + + if (filepos == HA_OFFSET_ERROR) + { + skipp_deleted_blocks=1; + if (info->lastpos == HA_OFFSET_ERROR) /* First read ? */ + filepos= info->s->pack.header_length; /* Read first record */ + else + filepos= info->nextpos; + } + + info->lastinx= -1; /* Can't forward or backward */ + /* Init all but update-flag */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + + DBUG_RETURN ((*info->s->read_rnd)(info,buf,filepos,skipp_deleted_blocks)); +} diff --git a/myisam/mi_rsame.c b/myisam/mi_rsame.c new file mode 100644 index 00000000000..a4092b53c0b --- /dev/null +++ b/myisam/mi_rsame.c @@ -0,0 +1,65 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "myisamdef.h" + + /* + ** Find current row with read on position or read on key + ** If inx >= 0 find record using key + ** Return values: + ** 0 = Ok. + ** HA_ERR_KEY_NOT_FOUND = Row is deleted + ** HA_ERR_END_OF_FILE = End of file + */ + + +int mi_rsame(MI_INFO *info, byte *record, int inx) +{ + DBUG_ENTER("mi_rsame"); + + if (inx != -1 && ! (((ulonglong) 1 << inx) & info->s->state.key_map)) + { + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + } + if (info->lastpos == HA_OFFSET_ERROR || info->update & HA_STATE_DELETED) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); /* No current record */ + } + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + /* Read row from data file */ + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(my_errno); + + if (inx >= 0) + { + info->lastinx=inx; + info->lastkey_length=_mi_make_key(info,(uint) inx,info->lastkey,record, + info->lastpos); + if (info->s->concurrent_insert) + rw_rdlock(&info->s->key_root_lock[inx]); + VOID(_mi_search(info,info->s->keyinfo+inx,info->lastkey,0,SEARCH_SAME, + info->s->state.key_root[inx])); + if (info->s->concurrent_insert) + rw_unlock(&info->s->key_root_lock[inx]); + } + + if (!(*info->read_record)(info,info->lastpos,record)) + DBUG_RETURN(0); + if (my_errno == HA_ERR_RECORD_DELETED) + my_errno=HA_ERR_KEY_NOT_FOUND; + DBUG_RETURN(my_errno); +} /* mi_rsame */ diff --git a/myisam/mi_rsamepos.c b/myisam/mi_rsamepos.c new file mode 100644 index 00000000000..28a5b6783b2 --- /dev/null +++ b/myisam/mi_rsamepos.c @@ -0,0 +1,56 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* read record through position and fix key-position */ +/* As mi_rsame but supply a position */ + +#include "myisamdef.h" + + + /* + ** If inx >= 0 update index pointer + ** Returns one of the following values: + ** 0 = Ok. + ** HA_ERR_KEY_NOT_FOUND = Row is deleted + ** HA_ERR_END_OF_FILE = End of file + */ + +int mi_rsame_with_pos(MI_INFO *info, byte *record, int inx, my_off_t filepos) +{ + DBUG_ENTER("mi_rsame_with_pos"); + + if (inx < -1 || ! (((ulonglong) 1 << inx) & info->s->state.key_map)) + { + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + } + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + if ((*info->s->read_rnd)(info,record,filepos,0)) + { + if (my_errno == HA_ERR_RECORD_DELETED) + my_errno=HA_ERR_KEY_NOT_FOUND; + DBUG_RETURN(my_errno); + } + info->lastpos=filepos; + info->lastinx=inx; + if (inx >= 0) + { + info->lastkey_length=_mi_make_key(info,(uint) inx,info->lastkey,record, + info->lastpos); + info->update|=HA_STATE_KEY_CHANGED; /* Don't use indexposition */ + } + DBUG_RETURN(0); +} /* mi_rsame_pos */ diff --git a/myisam/mi_scan.c b/myisam/mi_scan.c new file mode 100644 index 00000000000..c06f092ab17 --- /dev/null +++ b/myisam/mi_scan.c @@ -0,0 +1,46 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Read through all rows sequntially */ + +#include "myisamdef.h" + +int mi_scan_init(register MI_INFO *info) +{ + DBUG_ENTER("mi_scan_init"); + info->nextpos=info->s->pack.header_length; /* Read first record */ + info->lastinx= -1; /* Can't forward or backward */ + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + DBUG_RETURN(0); +} + +/* + Read a row based on position. + If filepos= HA_OFFSET_ERROR then read next row + Return values + Returns one of following values: + 0 = Ok. + HA_ERR_END_OF_FILE = EOF. +*/ + +int mi_scan(MI_INFO *info, byte *buf) +{ + DBUG_ENTER("mi_scan"); + /* Init all but update-flag */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + DBUG_RETURN ((*info->s->read_rnd)(info,buf,info->nextpos,1)); +} diff --git a/myisam/mi_search.c b/myisam/mi_search.c new file mode 100644 index 00000000000..e7a654c4da8 --- /dev/null +++ b/myisam/mi_search.c @@ -0,0 +1,1877 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* key handling functions */ + +#include "fulltext.h" +#include "m_ctype.h" + +#define CMP(a,b) (a<b ? -1 : a == b ? 0 : 1) + +static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, + uint *return_key_length); + + /* Check index */ + +int _mi_check_index(MI_INFO *info, int inx) +{ + if (inx == -1) /* Use last index */ + inx=info->lastinx; + if (inx < 0 || ! (((ulonglong) 1 << inx) & info->s->state.key_map)) + { + my_errno=HA_ERR_WRONG_INDEX; + return -1; + } + if (info->lastinx != inx) /* Index changed */ + { + info->lastinx = inx; + info->page_changed=1; + info->update= ((info->update & (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED)) | + HA_STATE_NEXT_FOUND | HA_STATE_PREV_FOUND); + } + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + return(-1); + return(inx); +} /* mi_check_index */ + + + /* + ** Search after row by a key + ** Position to row is stored in info->lastpos + ** Return: -1 if not found + ** 1 if one should continue search on higher level + */ + +int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_len, uint nextflag, register my_off_t pos) +{ + my_bool last_key; + int error,flag; + uint nod_flag; + uchar *keypos,*maxpos; + uchar lastkey[MI_MAX_KEY_BUFF],*buff; + DBUG_ENTER("_mi_search"); + DBUG_PRINT("enter",("pos: %ld nextflag: %d lastpos: %ld", + pos,nextflag,info->lastpos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_len);); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + info->lastpos= HA_OFFSET_ERROR; + if (!(nextflag & (SEARCH_SMALLER | SEARCH_BIGGER | SEARCH_LAST))) + DBUG_RETURN(-1); /* Not found ; return error */ + DBUG_RETURN(1); /* Search at upper levels */ + } + + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,info->buff, + test(!(nextflag & SEARCH_SAVE_BUFF))))) + goto err; + DBUG_DUMP("page",(byte*) buff,mi_getint(buff)); + + flag=(*keyinfo->bin_search)(info,keyinfo,buff,key,key_len,nextflag, + &keypos,lastkey, &last_key); + if (flag == MI_FOUND_WRONG_KEY) + DBUG_RETURN(-1); + nod_flag=mi_test_if_nod(buff); + maxpos=buff+mi_getint(buff)-1; + + if (flag) + { + if ((error=_mi_search(info,keyinfo,key,key_len,nextflag, + _mi_kpos(nod_flag,keypos))) <= 0) + DBUG_RETURN(error); + + if (flag >0) + { + if (nextflag & (SEARCH_SMALLER | SEARCH_LAST) && + keypos == buff+2+nod_flag) + DBUG_RETURN(1); /* Bigger than key */ + } + else if (nextflag & SEARCH_BIGGER && keypos >= maxpos) + DBUG_RETURN(1); /* Smaller than key */ + } + else + { + if (nextflag & SEARCH_FIND && (!(keyinfo->flag & HA_NOSAME) + || key_len) && nod_flag) + { + if ((error=_mi_search(info,keyinfo,key,key_len,SEARCH_FIND, + _mi_kpos(nod_flag,keypos))) >= 0 || + my_errno != HA_ERR_KEY_NOT_FOUND) + DBUG_RETURN(error); + info->last_keypage= HA_OFFSET_ERROR; /* Buffer not in memory */ + } + } + if (pos != info->last_keypage) + { + uchar *old_buff=buff; + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,info->buff, + test(!(nextflag & SEARCH_SAVE_BUFF))))) + goto err; + keypos=buff+(keypos-old_buff); + maxpos=buff+(maxpos-old_buff); + } + + if ((nextflag & (SEARCH_SMALLER | SEARCH_LAST)) && flag != 0) + { + uint not_used; + if (_mi_get_prev_key(info,keyinfo, buff, info->lastkey, keypos, + &info->lastkey_length)) + goto err; + if ((nextflag & SEARCH_LAST) && + _mi_key_cmp(keyinfo->seg, info->lastkey, key, key_len, SEARCH_FIND, + ¬_used)) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + goto err; + } + } + else + { + info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey); + if (!info->lastkey_length) + goto err; + memcpy(info->lastkey,lastkey,info->lastkey_length); + } + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + /* Save position for a possible read next / previous */ + info->int_keypos=info->buff+ (keypos-buff); + info->int_maxpos=info->buff+ (maxpos-buff); + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=0; + info->buff_used= (info->buff != buff); /* If we have to reread buff */ + + DBUG_PRINT("exit",("found key at %ld",info->lastpos)); + DBUG_RETURN(0); +err: + DBUG_PRINT("exit",("Error: %d",my_errno)); + info->lastpos= HA_OFFSET_ERROR; + info->page_changed=1; + DBUG_RETURN (-1); +} /* _mi_search */ + + + /* Search after key in page-block */ + /* If packed key puts smaller or identical key in buff */ + /* ret_pos point to where find or bigger key starts */ + /* ARGSUSED */ + +int _mi_bin_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff __attribute__((unused)), my_bool *last_key) +{ + reg4 int start,mid,end,save_end; + int flag; + uint totlength,nod_flag,not_used; + DBUG_ENTER("_mi_bin_search"); + + LINT_INIT(flag); + totlength=keyinfo->keylength+(nod_flag=mi_test_if_nod(page)); + start=0; mid=1; + save_end=end=(int) ((mi_getint(page)-2-nod_flag)/totlength-1); + DBUG_PRINT("test",("mi_getint: %d end: %d",mi_getint(page),end)); + page+=2+nod_flag; + + while (start != end) + { + mid= (start+end)/2; + if ((flag=_mi_key_cmp(keyinfo->seg,page+(uint) mid*totlength,key,key_len, + comp_flag,¬_used)) + >= 0) + end=mid; + else + start=mid+1; + } + if (mid != start) + flag=_mi_key_cmp(keyinfo->seg,page+(uint) start*totlength,key,key_len, + comp_flag,¬_used); + if (flag < 0) + start++; /* point at next, bigger key */ + *ret_pos=page+(uint) start*totlength; + *last_key= end == save_end; + DBUG_PRINT("exit",("flag: %d keypos: %d",flag,start)); + DBUG_RETURN(flag); +} /* _mi_bin_search */ + + + /* Used instead of _mi_bin_search() when key is packed */ + /* Puts smaller or identical key in buff */ + /* Key is searched sequentially */ + +int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff, my_bool *last_key) +{ + int flag; + uint nod_flag,length,not_used; + uchar t_buff[MI_MAX_KEY_BUFF],*end; + DBUG_ENTER("_mi_seq_search"); + + LINT_INIT(flag); LINT_INIT(length); + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + *ret_pos=page; + t_buff[0]=0; /* Avoid bugs */ + while (page < end) + { + length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,t_buff); + if (length == 0 || page > end) + { + my_errno=HA_ERR_CRASHED; + DBUG_PRINT("error",("Found wrong key: length: %d page: %lx end: %lx", + length,page,end)); + DBUG_RETURN(MI_FOUND_WRONG_KEY); + } + if ((flag=_mi_key_cmp(keyinfo->seg,t_buff,key,key_len,comp_flag, + ¬_used)) >= 0) + break; +#ifdef EXTRA_DEBUG + DBUG_PRINT("loop",("page: %lx key: '%s' flag: %d",page,t_buff,flag)); +#endif + memcpy(buff,t_buff,length); + *ret_pos=page; + } + if (flag == 0) + memcpy(buff,t_buff,length); /* Result is first key */ + *last_key= page == end; + DBUG_PRINT("exit",("flag: %d ret_pos: %lx",flag,*ret_pos)); + DBUG_RETURN(flag); +} /* _mi_seq_search */ + + + /* Get pos to a key_block */ + +my_off_t _mi_kpos(uint nod_flag, uchar *after_key) +{ + after_key-=nod_flag; + switch (nod_flag) { +#if SIZEOF_OFF_T > 4 + case 7: + return mi_uint7korr(after_key)*MI_KEY_BLOCK_LENGTH; + case 6: + return mi_uint6korr(after_key)*MI_KEY_BLOCK_LENGTH; + case 5: + return mi_uint5korr(after_key)*MI_KEY_BLOCK_LENGTH; +#else + case 7: + after_key++; + case 6: + after_key++; + case 5: + after_key++; +#endif + case 4: + return ((my_off_t) mi_uint4korr(after_key))*MI_KEY_BLOCK_LENGTH; + case 3: + return ((my_off_t) mi_uint3korr(after_key))*MI_KEY_BLOCK_LENGTH; + case 2: + return (my_off_t) (mi_uint2korr(after_key)*MI_KEY_BLOCK_LENGTH); + case 1: + return (uint) (*after_key)*MI_KEY_BLOCK_LENGTH; + case 0: /* At leaf page */ + default: /* Impossible */ + return(HA_OFFSET_ERROR); + } +} /* _kpos */ + + + /* Save pos to a key_block */ + +void _mi_kpointer(register MI_INFO *info, register uchar *buff, my_off_t pos) +{ + pos/=MI_KEY_BLOCK_LENGTH; + switch (info->s->base.key_reflength) { +#if SIZEOF_OFF_T > 4 + case 7: mi_int7store(buff,pos); break; + case 6: mi_int6store(buff,pos); break; + case 5: mi_int5store(buff,pos); break; +#else + case 7: *buff++=0; + /* fall trough */ + case 6: *buff++=0; + /* fall trough */ + case 5: *buff++=0; + /* fall trough */ +#endif + case 4: mi_int4store(buff,pos); break; + case 3: mi_int3store(buff,pos); break; + case 2: mi_int2store(buff,(uint) pos); break; + case 1: buff[0]= (uchar) pos; break; + default: abort(); /* impossible */ + } +} /* _mi_kpointer */ + + + /* Calc pos to a data-record from a key */ + + +my_off_t _mi_dpos(MI_INFO *info, uint nod_flag, uchar *after_key) +{ + my_off_t pos; + after_key-=(nod_flag + info->s->rec_reflength); + switch (info->s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: pos= (my_off_t) mi_uint5korr(after_key); break; + case 7: pos= (my_off_t) mi_uint7korr(after_key); break; + case 6: pos= (my_off_t) mi_uint6korr(after_key); break; + case 5: pos= (my_off_t) mi_uint5korr(after_key); break; +#else + case 8: pos= (my_off_t) mi_uint4korr(after_key+4); break; + case 7: pos= (my_off_t) mi_uint4korr(after_key+3); break; + case 6: pos= (my_off_t) mi_uint4korr(after_key+2); break; + case 5: pos= (my_off_t) mi_uint4korr(after_key+1); break; +#endif + case 4: pos= (my_off_t) mi_uint4korr(after_key); break; + case 3: pos= (my_off_t) mi_uint3korr(after_key); break; + case 2: pos= (my_off_t) mi_uint2korr(after_key); break; + default: + pos=0L; /* Shut compiler up */ + } + return (info->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*info->s->base.pack_reclength; +} + + +/* Calc position from a record pointer ( in delete link chain ) */ + +my_off_t _mi_rec_pos(MYISAM_SHARE *s, uchar *ptr) +{ + my_off_t pos; + switch (s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: + pos= (my_off_t) mi_uint8korr(ptr); + if (pos == HA_OFFSET_ERROR) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 7: + pos= (my_off_t) mi_uint7korr(ptr); + if (pos == (((my_off_t) 1) << 56) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 6: + pos= (my_off_t) mi_uint6korr(ptr); + if (pos == (((my_off_t) 1) << 48) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 5: + pos= (my_off_t) mi_uint5korr(ptr); + if (pos == (((my_off_t) 1) << 40) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; +#else + case 8: + case 7: + case 6: + case 5: + ptr+= (s->rec_reflength-4); + /* fall through */ +#endif + case 4: + pos= (my_off_t) mi_uint4korr(ptr); + if (pos == (my_off_t) (uint32) ~0L) + return HA_OFFSET_ERROR; + break; + case 3: + pos= (my_off_t) mi_uint3korr(ptr); + if (pos == (my_off_t) (1 << 24) -1) + return HA_OFFSET_ERROR; + break; + case 2: + pos= (my_off_t) mi_uint2korr(ptr); + if (pos == (my_off_t) (1 << 16) -1) + return HA_OFFSET_ERROR; + break; + default: abort(); /* Impossible */ + } + return ((s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*s->base.pack_reclength); +} + + + /* save position to record */ + +void _mi_dpointer(MI_INFO *info, uchar *buff, my_off_t pos) +{ + if (!(info->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && + pos != HA_OFFSET_ERROR) + pos/=info->s->base.pack_reclength; + + switch (info->s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: mi_int8store(buff,pos); break; + case 7: mi_int7store(buff,pos); break; + case 6: mi_int6store(buff,pos); break; + case 5: mi_int5store(buff,pos); break; +#else + case 8: *buff++=0; + /* fall trough */ + case 7: *buff++=0; + /* fall trough */ + case 6: *buff++=0; + /* fall trough */ + case 5: *buff++=0; + /* fall trough */ +#endif + case 4: mi_int4store(buff,pos); break; + case 3: mi_int3store(buff,pos); break; + case 2: mi_int2store(buff,(uint) pos); break; + default: abort(); /* Impossible */ + } +} /* _mi_dpointer */ + + +int _mi_compare_text(CHARSET_INFO *charset_info, uchar *a, uint a_length, + uchar *b, uint b_length, my_bool part_key) +{ + uint length= min(a_length,b_length); + uchar *end= a+ length; + int flag; + +#ifdef USE_STRCOLL + if (use_strcoll(charset_info)) + { + if ((flag = my_strnncoll(charset_info, a, a_length, b, b_length))) + return flag; + } + else +#endif + { + uchar *sort_order=charset_info->sort_order; + while (a < end) + if ((flag= (int) sort_order[*a++] - (int) sort_order[*b++])) + return flag; + } + if (part_key && b_length < a_length) + return 0; + return (int) (a_length-b_length); +} + + +static int compare_bin(uchar *a, uint a_length, uchar *b, uint b_length, + my_bool part_key) +{ + uint length= min(a_length,b_length); + uchar *end= a+ length; + int flag; + + while (a < end) + if ((flag= (int) *a++ - (int) *b++)) + return flag; + if (part_key && b_length < a_length) + return 0; + return (int) (a_length-b_length); +} + + + /* + ** Compare two keys with is bigger + ** Returns <0, 0, >0 acording to with is bigger + ** Key_length specifies length of key to use. Number-keys can't + ** be splited + ** If flag <> SEARCH_FIND compare also position + */ + +#define FCMP(A,B) ((int) (A) - (int) (B)) + +int _mi_key_cmp(register MI_KEYSEG *keyseg, register uchar *a, + register uchar *b, uint key_length, uint nextflag, + uint *diff_pos) +{ + int flag; + int16 s_1,s_2; + int32 l_1,l_2; + uint32 u_1,u_2; + float f_1,f_2; + double d_1,d_2; + uint next_key_length; + + if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) + key_length=USE_WHOLE_KEY; + *diff_pos=0; + + for ( ; (int) key_length >0 ; key_length=next_key_length, keyseg++) + { + uchar *end; + (*diff_pos)++; + + /* Handle NULL part */ + if (keyseg->null_bit) + { + key_length--; + if (*a != *b) + { + flag = (int) *a - (int) *b; + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + } + b++; + if (!*a++) /* If key was NULL */ + { + if (nextflag == (SEARCH_FIND | SEARCH_UPDATE)) + nextflag=SEARCH_SAME; /* Allow duplicate keys */ + next_key_length=key_length; + continue; /* To next key part */ + } + } + end= a+ min(keyseg->length,key_length); + next_key_length=key_length-keyseg->length; + + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_TEXT: /* Ascii; Key is converted */ + if (keyseg->flag & HA_SPACE_PACK) + { + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if ((flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; + } + else + { + uint length=(uint) (end-a); + if ((flag=_mi_compare_text(keyseg->charset,a,length,b,length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a=end; + b+=length; + } + break; + case HA_KEYTYPE_BINARY: + if (keyseg->flag & HA_SPACE_PACK) + { + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if ((flag=compare_bin(a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; + } + else + { + uint length=keyseg->length; + if ((flag=compare_bin(a,length,b,length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=length; + b+=length; + } + break; + case HA_KEYTYPE_VARTEXT: + { + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if ((flag=_mi_compare_text(keyseg->charset,a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; + } + break; + case HA_KEYTYPE_VARBINARY: + { + int a_length,b_length,pack_length; + get_key_length(a_length,a); + get_key_pack_length(b_length,pack_length,b); + next_key_length=key_length-b_length-pack_length; + + if ((flag=compare_bin(a,a_length,b,b_length, + (my_bool) ((nextflag & SEARCH_PREFIX) && + next_key_length <= 0)))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a+=a_length; + b+=b_length; + break; + } + break; + case HA_KEYTYPE_INT8: + { + int i_1= (int) *((signed char*) a); + int i_2= (int) *((signed char*) b); + if ((flag = CMP(i_1,i_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b++; + break; + } + case HA_KEYTYPE_SHORT_INT: + s_1= mi_sint2korr(a); + s_2= mi_sint2korr(b); + if ((flag = CMP(s_1,s_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 2; /* sizeof(short int); */ + break; + case HA_KEYTYPE_USHORT_INT: + { + uint16 us_1,us_2; + us_1= mi_sint2korr(a); + us_2= mi_sint2korr(b); + if ((flag = CMP(us_1,us_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+=2; /* sizeof(short int); */ + break; + } + case HA_KEYTYPE_LONG_INT: + l_1= mi_sint4korr(a); + l_2= mi_sint4korr(b); + if ((flag = CMP(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 4; /* sizeof(long int); */ + break; + case HA_KEYTYPE_ULONG_INT: + u_1= mi_sint4korr(a); + u_2= mi_sint4korr(b); + if ((flag = CMP(u_1,u_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 4; /* sizeof(long int); */ + break; + case HA_KEYTYPE_INT24: + l_1=mi_sint3korr(a); + l_2=mi_sint3korr(b); + if ((flag = CMP(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 3; + break; + case HA_KEYTYPE_UINT24: + l_1=mi_uint3korr(a); + l_2=mi_uint3korr(b); + if ((flag = CMP(l_1,l_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 3; + break; + case HA_KEYTYPE_FLOAT: + mi_float4get(f_1,a); + mi_float4get(f_2,b); + if ((flag = CMP(f_1,f_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 4; /* sizeof(float); */ + break; + case HA_KEYTYPE_DOUBLE: + mi_float8get(d_1,a); + mi_float8get(d_2,b); + if ((flag = CMP(d_1,d_2))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 8; /* sizeof(double); */ + break; + case HA_KEYTYPE_NUM: /* Numeric key */ + { + int swap_flag= 0; + int alength,blength; + + if (keyseg->flag & HA_REVERSE_SORT) + { + swap(uchar*,a,b); + swap_flag=1; /* Remember swap of a & b */ + end= a+ (int) (end-b); + } + if (keyseg->flag & HA_SPACE_PACK) + { + alength= *a++; blength= *b++; + end=a+alength; + } + else + { + alength= (int) (end-a); + blength=keyseg->length; + /* remove pre space from keys */ + for ( ; alength && *a == ' ' ; a++, alength--) ; + for ( ; blength && *b == ' ' ; b++, blength--) ; + } + + if (*a == '-') + { + if (*b != '-') + return -1; + a++; b++; + swap(uchar*,a,b); + swap(int,alength,blength); + swap_flag=1-swap_flag; + alength--; blength--; + end=a+alength; + } + else if (*b == '-') + return 1; + while (alength && (*a == '+' || *a == '0')) + { + a++; alength--; + } + while (blength && (*b == '+' || *b == '0')) + { + b++; blength--; + } + if (alength != blength) + return (alength < blength) ? -1 : 1; + while (a < end) + if (*a++ != *b++) + return ((int) a[-1] - (int) b[-1]); + + if (swap_flag) /* Restore pointers */ + swap(uchar*,a,b); + break; + } +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + { + longlong ll_a,ll_b; + ll_a= mi_sint8korr(a); + ll_b= mi_sint8korr(b); + if ((flag = CMP(ll_a,ll_b))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 8; + break; + } + case HA_KEYTYPE_ULONGLONG: + { + ulonglong ll_a,ll_b; + ll_a= mi_uint8korr(a); + ll_b= mi_uint8korr(b); + if ((flag = CMP(ll_a,ll_b))) + return ((keyseg->flag & HA_REVERSE_SORT) ? -flag : flag); + a= end; + b+= 8; + break; + } +#endif + case HA_KEYTYPE_END: /* Ready */ + goto end; /* diff_pos is incremented */ + } + } + (*diff_pos)++; +end: + if (!(nextflag & SEARCH_FIND)) + { + uint i; + if (nextflag & (SEARCH_NO_FIND | SEARCH_LAST)) /* Find record after key */ + return (nextflag & (SEARCH_BIGGER | SEARCH_LAST)) ? -1 : 1; + flag=0; + for (i=keyseg->length ; i-- > 0 ; ) + { + if (*a++ != *b++) + { + flag= FCMP(a[-1],b[-1]); + break; + } + } + if (nextflag & SEARCH_SAME) + return (flag); /* read same */ + if (nextflag & SEARCH_BIGGER) + return (flag <= 0 ? -1 : 1); /* read next */ + return (flag < 0 ? -1 : 1); /* read previous */ + } + return 0; +} /* _mi_key_cmp */ + + + /* Get key from key-block */ + /* page points at previous key; its advanced to point at next key */ + /* key should contain previous key */ + /* Returns length of found key + pointers */ + /* nod_flag is a flag if we are on nod */ + + /* same as _mi_get_key but used with fixed length keys */ + +uint _mi_get_static_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page, register uchar *key) +{ + memcpy((byte*) key,(byte*) *page, + (size_t) (keyinfo->keylength+nod_flag)); + *page+=keyinfo->keylength+nod_flag; + return(keyinfo->keylength); +} /* _mi_get_static_key */ + + + +uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page_pos, register uchar *key) +{ + reg1 MI_KEYSEG *keyseg; + uchar *start_key,*page=*page_pos; + uint length; + + start_key=key; + for (keyseg=keyinfo->seg ; keyseg->type ;keyseg++) + { + /* First key part is always packed !*/ + if (keyseg->flag & HA_PACK_KEY) + { + /* key with length, packed to previous key */ + uchar *start=key; + uint packed= *page & 128,tot_length,rest_length; + if (keyseg->length >= 127) + { + length=mi_uint2korr(page) & 32767; + page+=2; + } + else + length= *page++ & 127; + + if (packed) + { + if (length > (uint) keyseg->length) + { + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ + } + if (length == 0) /* Same key */ + { + if (keyseg->flag & HA_NULL_PART) + *key++=1; /* Can't be NULL */ + get_key_length(length,key); + key+= length; /* Same diff_key as prev */ + if (length > keyseg->length) + { + DBUG_PRINT("error",("Found too long null packed key: %d of %d at %lx", + length, keyseg->length, *page_pos)); + DBUG_DUMP("key",(char*) *page_pos,16); + my_errno=HA_ERR_CRASHED; + return 0; + } + continue; + } + if (keyseg->flag & HA_NULL_PART) + key++; /* Skipp null marker*/ + + get_key_length(rest_length,page); + tot_length=rest_length+length; + + /* If the stored length has changed, we must move the key */ + if (tot_length >= 255 && *start != 255) + { + /* length prefix changed from a length of one to a length of 3 */ + bmove_upp((char*) key+length+3,(char*) key+length+1,length); + *key=255; + mi_int2store(key+1,tot_length); + key+=3+length; + } + else if (tot_length < 255 && *start == 255) + { + bmove(key+1,key+3,length); + *key=tot_length; + key+=1+length; + } + else + { + store_key_length_inc(key,tot_length); + key+=length; + } + memcpy(key,page,rest_length); + page+=rest_length; + key+=rest_length; + continue; + } + else + { + if (keyseg->flag & HA_NULL_PART) + { + if (!length--) /* Null part */ + { + *key++=0; + continue; + } + *key++=1; /* Not null */ + } + } + if (length > (uint) keyseg->length) + { + DBUG_PRINT("error",("Found too long packed key: %d of %d at %lx", + length, keyseg->length, *page_pos)); + DBUG_DUMP("key",(char*) *page_pos,16); + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ + } + store_key_length_inc(key,length); + } + else + { + if (keyseg->flag & HA_NULL_PART) + { + if (!(*key++ = *page++)) + continue; + } + if (keyseg->flag & + (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) + { + uchar *tmp=page; + get_key_length(length,tmp); + length+=(uint) (tmp-page); + } + else + length=keyseg->length; + } + memcpy((byte*) key,(byte*) page,(size_t) length); + key+=length; + page+=length; + } + length=keyseg->length+nod_flag; + bmove((byte*) key,(byte*) page,length); + *page_pos= page+length; + return ((uint) (key-start_key)+keyseg->length); +} /* _mi_get_pack_key */ + + + +/* key that is packed relatively to previous */ + +uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page_pos, register uchar *key) +{ + reg1 MI_KEYSEG *keyseg; + uchar *start_key,*page=*page_pos,*page_end,*from,*from_end; + uint length,tmp; + + page_end=page+MI_MAX_KEY_BUFF+1; + start_key=key; + + get_key_length(length,page); + if (length) + { + if (length > keyinfo->maxlength) + { + DBUG_PRINT("error",("Found too long binary packed key: %d of %d at %lx", + length, keyinfo->maxlength, *page_pos)); + DBUG_DUMP("key",(char*) *page_pos,16); + my_errno=HA_ERR_CRASHED; + return 0; /* Wrong key */ + } + from=key; from_end=key+length; + } + else + { + from=page; from_end=page_end; /* Not packed key */ + } + + /* + The trouble is that key is split in two parts: + The first part is in from ...from_end-1. + The second part starts at page + */ + for (keyseg=keyinfo->seg ; keyseg->type ;keyseg++) + { + if (keyseg->flag & HA_NULL_PART) + { + if (from == from_end) { from=page; from_end=page_end; } + if (!(*key++ = *from++)) + continue; /* Null part */ + } + if (keyseg->flag & (HA_VAR_LENGTH | HA_BLOB_PART | HA_SPACE_PACK)) + { + /* Get length of dynamic length key part */ + if (from == from_end) { from=page; from_end=page_end; } + if ((length= (*key++ = *from++)) == 255) + { + if (from == from_end) { from=page; from_end=page_end; } + length= (uint) ((*key++ = *from++)) << 8; + if (from == from_end) { from=page; from_end=page_end; } + length+= (uint) ((*key++ = *from++)); + } + } + else + length=keyseg->length; + + if ((tmp=(uint) (from_end-from)) <= length) + { + key+=tmp; /* Use old key */ + length-=tmp; + from=page; from_end=page_end; + } + memcpy((byte*) key,(byte*) from,(size_t) length); + key+=length; + from+=length; + } + length=keyseg->length+nod_flag; + if ((tmp=(uint) (from_end-from)) <= length) + { + memcpy(key+tmp,page,length-tmp); /* Get last part of key */ + *page_pos= page+length-tmp; + } + else + { + if (from_end != page_end) + { + DBUG_PRINT("error",("Error when unpacking key")); + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ + } + memcpy((byte*) key,(byte*) from,(size_t) length); + *page_pos= from+length; + } + return((uint) (key-start_key)+keyseg->length); +} + + + /* Get key at position without knowledge of previous key */ + /* Returns pointer to next key */ + +uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, uint *return_key_length) +{ + uint nod_flag; + DBUG_ENTER("_mi_get_key"); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + bmove((byte*) key,(byte*) keypos,keyinfo->keylength+nod_flag); + DBUG_RETURN(keypos+keyinfo->keylength+nod_flag); + } + else + { + page+=2+nod_flag; + key[0]=0; /* safety */ + while (page <= keypos) + { + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); + if (*return_key_length == 0) + { + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } + } + DBUG_PRINT("exit",("page: %lx length: %d",page,*return_key_length)); + DBUG_RETURN(page); +} /* _mi_get_key */ + + + /* Get key at position without knowledge of previous key */ + /* Returns 0 if ok */ + +static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, + uint *return_key_length) +{ + uint nod_flag; + DBUG_ENTER("_mi_get_prev_key"); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + *return_key_length=keyinfo->keylength; + bmove((byte*) key,(byte*) keypos- *return_key_length-nod_flag, + *return_key_length); + DBUG_RETURN(0); + } + else + { + page+=2+nod_flag; + key[0]=0; /* safety */ + while (page < keypos) + { + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); + if (*return_key_length == 0) + { + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(1); + } + } + } + DBUG_RETURN(0); +} /* _mi_get_key */ + + + + /* Get last key from key-page */ + /* Return pointer to where key starts */ + +uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *lastkey, uchar *endpos, uint *return_key_length) +{ + uint nod_flag; + uchar *lastpos; + DBUG_ENTER("_mi_get_last_key"); + DBUG_PRINT("enter",("page: %lx endpos: %lx",page,endpos)); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + lastpos=endpos-keyinfo->keylength-nod_flag; + *return_key_length=keyinfo->keylength; + if (lastpos > page) + bmove((byte*) lastkey,(byte*) lastpos,keyinfo->keylength+nod_flag); + } + else + { + lastpos=(page+=2+nod_flag); + lastkey[0]=0; + while (page < endpos) + { + lastpos=page; + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,lastkey); + if (*return_key_length == 0) + { + DBUG_PRINT("error",("Couldn't find last key: page: %lx",page)); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } + } + DBUG_PRINT("exit",("lastpos: %lx length: %d",lastpos,*return_key_length)); + DBUG_RETURN(lastpos); +} /* _mi_get_last_key */ + + + /* Calculate length of key */ + +uint _mi_keylength(MI_KEYDEF *keyinfo, register uchar *key) +{ + reg1 MI_KEYSEG *keyseg; + uchar *start; + + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + return (keyinfo->keylength); + + start=key; + for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++) + { + if (keyseg->flag & HA_NULL_PART) + if (!*key++) + continue; + if (keyseg->flag & (HA_SPACE_PACK | HA_BLOB_PART | HA_VAR_LENGTH)) + { + uint length; + get_key_length(length,key); + key+=length; + } + else + key+= keyseg->length; + } + return((uint) (key-start)+keyseg->length); +} /* _mi_keylength */ + + + /* Move a key */ + +uchar *_mi_move_key(MI_KEYDEF *keyinfo, uchar *to, uchar *from) +{ + reg1 uint length; + memcpy((byte*) to, (byte*) from, + (size_t) (length=_mi_keylength(keyinfo,from))); + return to+length; +} + + /* Find next/previous record with same key */ + /* This can't be used when database is touched after last read */ + +int _mi_search_next(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_length, uint nextflag, my_off_t pos) +{ + int error; + uint nod_flag; + uchar lastkey[MI_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_search_next"); + DBUG_PRINT("enter",("nextflag: %d lastpos: %ld int_keypos: %lx", + nextflag,(long) info->lastpos,info->int_keypos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_length);); + + /* Force full read if we are at last key or if we are not on a leaf + and the key tree has changed since we used it last time */ + + if (((nextflag & SEARCH_BIGGER) && info->int_keypos >= info->int_maxpos) || + info->page_changed || + (info->int_keytree_version != keyinfo->version && + (info->int_nod_flag || info->buff_used))) + DBUG_RETURN(_mi_search(info,keyinfo,key,key_length, + nextflag | SEARCH_SAVE_BUFF, pos)); + + if (info->buff_used) + { + if (!_mi_fetch_keypage(info,keyinfo,info->last_search_keypage, + info->buff,0)) + DBUG_RETURN(-1); + info->buff_used=0; + } + + /* Last used buffer is in info->buff */ + nod_flag=mi_test_if_nod(info->buff); + memcpy(lastkey,key,key_length); + + if (nextflag & SEARCH_BIGGER) /* Next key */ + { + my_off_t tmp_pos=_mi_kpos(nod_flag,info->int_keypos); + if (tmp_pos != HA_OFFSET_ERROR) + { + if ((error=_mi_search(info,keyinfo,key,key_length, + nextflag | SEARCH_SAVE_BUFF, tmp_pos)) <=0) + DBUG_RETURN(error); + } + if (!(info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag, + &info->int_keypos,lastkey))) + DBUG_RETURN(-1); + } + else /* Previous key */ + { + uint length; + /* Find start of previous key */ + info->int_keypos=_mi_get_last_key(info,keyinfo,info->buff,lastkey, + info->int_keypos, &length); + if (!info->int_keypos) + DBUG_RETURN(-1); + if (info->int_keypos == info->buff+2) + DBUG_RETURN(_mi_search(info,keyinfo,key,key_length, + nextflag | SEARCH_SAVE_BUFF, pos)); + if ((error=_mi_search(info,keyinfo,key,0,nextflag | SEARCH_SAVE_BUFF, + _mi_kpos(nod_flag,info->int_keypos))) <= 0) + DBUG_RETURN(error); + + if (! _mi_get_last_key(info,keyinfo,info->buff,lastkey, + info->int_keypos,&info->lastkey_length)) + DBUG_RETURN(-1); + } + memcpy(info->lastkey,lastkey,info->lastkey_length); + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + DBUG_PRINT("exit",("found key at %d",info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_next */ + + + /* Search after position for the first row in an index */ + /* This is stored in info->lastpos */ + +int _mi_search_first(register MI_INFO *info, register MI_KEYDEF *keyinfo, + register my_off_t pos) +{ + uint nod_flag; + uchar *page; + DBUG_ENTER("_mi_search_first"); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + + do + { + if (!_mi_fetch_keypage(info,keyinfo,pos,info->buff,0)) + { + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + nod_flag=mi_test_if_nod(info->buff); + page=info->buff+2+nod_flag; + } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); + + info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page, + info->lastkey); + info->int_keypos=page; info->int_maxpos=info->buff+mi_getint(info->buff)-1; + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=info->buff_used=0; + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + + DBUG_PRINT("exit",("found key at %d",info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_first */ + + + /* Search after position for the last row in an index */ + /* This is stored in info->lastpos */ + +int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, + register my_off_t pos) +{ + uint nod_flag; + uchar *buff,*page; + DBUG_ENTER("_mi_search_last"); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + + buff=info->buff; + do + { + if (!_mi_fetch_keypage(info,keyinfo,pos,buff,0)) + { + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + page= buff+mi_getint(buff); + nod_flag=mi_test_if_nod(buff); + } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); + + if (!_mi_get_last_key(info,keyinfo,buff,info->lastkey,page, + &info->lastkey_length)) + DBUG_RETURN(-1); + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + info->int_keypos=info->int_maxpos=page; + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=info->buff_used=0; + + DBUG_PRINT("exit",("found key at %d",info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_last */ + + + +/**************************************************************************** +** +** Functions to store and pack a key in a page +** +** mi_calc_xx_key_length takes the following arguments: +** nod_flag If nod: Length of nod-pointer +** next_key Position to pos after the new key in buffer +** org_key Key that was before the next key in buffer +** prev_key Last key before current key +** key Key that will be stored +** s_temp Information how next key will be packed +****************************************************************************/ + +/* Static length key */ + +int +_mi_calc_static_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) +{ + s_temp->key=key; + return (int) (s_temp->totlength=keyinfo->keylength+nod_flag); +} + +/* Variable length key */ + +int +_mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) +{ + s_temp->key=key; + return (int) (s_temp->totlength=_mi_keylength(keyinfo,key)+nod_flag); +} + +/* + length of key with a variable length first segment which is prefix + compressed (myisamchk reports 'packed + stripped') + + Keys are compressed the following way: + + If the max length of first key segment <= 127 characters the prefix is + 1 byte else its 2 byte + + prefix byte The high bit is set if this is a prefix for the prev key + length Packed length if the previous was a prefix byte + [length] Length character of data + next-key-seg Next key segments + + If the first segment can have NULL: + The length is 0 for NULLS and 1+length for not null columns. + +*/ + +int +_mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) +{ + reg1 MI_KEYSEG *keyseg; + int length; + uint key_length,ref_length,org_key_length=0, + length_pack,new_key_length,diff_flag,pack_marker; + uchar *start,*end,*key_end,*sort_order; + bool same_length; + + length_pack=s_temp->ref_length=s_temp->n_ref_length=s_temp->n_length=0; + same_length=0; keyseg=keyinfo->seg; + key_length=_mi_keylength(keyinfo,key)+nod_flag; + + sort_order=0; + if ((keyinfo->flag & HA_FULLTEXT) && + ((keyseg->type == HA_KEYTYPE_TEXT) || + (keyseg->type == HA_KEYTYPE_VARTEXT)) && + !use_strcoll(keyseg->charset)) + sort_order=keyseg->charset->sort_order; + + /* diff flag contains how many bytes is needed to pack key */ + if (keyseg->length >= 127) + { + diff_flag=2; + pack_marker=32768; + } + else + { + diff_flag= 1; + pack_marker=128; + } + s_temp->pack_marker=pack_marker; + + /* Handle the case that the first part have NULL values */ + if (keyseg->flag & HA_NULL_PART) + { + if (!*key++) + { + s_temp->key=key; + s_temp->ref_length=s_temp->key_length=0; + s_temp->totlength=key_length-1+diff_flag; + s_temp->next_key_pos=0; /* No next key */ + return (s_temp->totlength); + } + s_temp->store_not_null=1; + key_length--; /* We don't store NULL */ + if (prev_key && !*prev_key++) + org_key=prev_key=0; /* Can't pack against prev */ + else if (org_key) + org_key++; /* Skipp NULL */ + } + else + s_temp->store_not_null=0; + s_temp->prev_key=org_key; + + /* The key part will start with a packed length */ + + get_key_pack_length(new_key_length,length_pack,key); + end=key_end= key+ new_key_length; + start=key; + + /* Calc how many characters are identical between this and the prev. key */ + if (prev_key) + { + get_key_length(org_key_length,prev_key); + s_temp->prev_key=prev_key; /* Pointer at data */ + /* Don't use key-pack if length == 0 */ + if (new_key_length && new_key_length == org_key_length) + same_length=1; + else if (new_key_length > org_key_length) + end=key+ org_key_length+1; + + if (sort_order) /* SerG */ + { + while (key < end && sort_order[*key] == sort_order[*prev_key]) + { + key++; prev_key++; + } + } + else + { + while (key < end && *key == *prev_key) + { + key++; prev_key++; + } + } + } + + s_temp->key=key; + s_temp->key_length= (uint) (key_end-key); + + if (same_length && key == key_end) + { + /* identical variable length key */ + s_temp->ref_length= pack_marker; + length=(int) key_length-(int) (key_end-start)-length_pack; + length+= diff_flag; + if (next_key) + { /* Can't combine with next */ + s_temp->n_length= *next_key; /* Needed by _mi_store_key */ + next_key=0; + } + } + else + { + if (start != key) + { /* Starts as prev key */ + ref_length= (uint) (key-start); + s_temp->ref_length= ref_length + pack_marker; + length= (int) (key_length - ref_length); + + length-= length_pack; + length+= diff_flag; + length+= ((new_key_length-ref_length) >= 255) ? 3 : 1;/* Rest_of_key */ + } + else + { + s_temp->key_length+=s_temp->store_not_null; /* If null */ + length= key_length - length_pack+ diff_flag; + } + } + s_temp->totlength=(uint) length; + s_temp->prev_length=0; + DBUG_PRINT("test",("tot_length: %d length: %d uniq_key_length: %d", + key_length,length,s_temp->key_length)); + + /* If something after that hasn't length=0, test if we can combine */ + if ((s_temp->next_key_pos=next_key)) + { + uint packed,n_length; + + packed = *next_key & 128; + if (diff_flag == 2) + { + n_length= mi_uint2korr(next_key) & 32767; /* Length of next key */ + next_key+=2; + } + else + n_length= *next_key++ & 127; + if (!packed) + n_length-= s_temp->store_not_null; + + if (n_length || packed) /* Don't pack 0 length keys */ + { + uint next_length_pack, new_ref_length=s_temp->ref_length; + + if (packed) + { + /* If first key and next key is packed (only on delete) */ + if (!prev_key && org_key) + { + get_key_length(org_key_length,org_key); + key=start; + if (sort_order) /* SerG */ + { + while (key < end && sort_order[*key] == sort_order[*org_key]) + { + key++; org_key++; + } + } + else + { + while (key < end && *key == *org_key) + { + key++; org_key++; + } + } + if ((new_ref_length= (key - start))) + new_ref_length+=pack_marker; + } + + if (!n_length) + { + /* + We put a different key between two identical variable length keys + Extend next key to have same prefix as this key + */ + if (new_ref_length) /* prefix of previus key */ + { /* make next key longer */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= org_key_length - + (new_ref_length-pack_marker); + s_temp->n_ref_length= s_temp->n_length= s_temp->prev_length; + n_length= get_pack_length(s_temp->prev_length); + s_temp->prev_key+= (new_ref_length - pack_marker); + length+= s_temp->prev_length + n_length; + } + else + { /* Can't use prev key */ + s_temp->part_of_prev_key=0; + s_temp->prev_length= org_key_length; + s_temp->n_ref_length=s_temp->n_length= org_key_length; + length+= org_key_length; + /* +get_pack_length(org_key_length); */ + } + return (int) length; + } + + ref_length=n_length; + get_key_pack_length(n_length,next_length_pack,next_key); + + /* Test if new keys has fewer characters that match the previous key */ + if (!new_ref_length) + { /* Can't use prev key */ + s_temp->part_of_prev_key= 0; + s_temp->prev_length= ref_length; + s_temp->n_ref_length= s_temp->n_length= n_length+ref_length; + /* s_temp->prev_key+= get_pack_length(org_key_length); */ + return (int) length+ref_length-next_length_pack; + } + if (ref_length+pack_marker > new_ref_length) + { + uint new_pack_length=new_ref_length-pack_marker; + /* We must copy characters from the original key to the next key */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= ref_length - new_pack_length; + s_temp->n_ref_length=s_temp->n_length=n_length + s_temp->prev_length; + s_temp->prev_key+= new_pack_length; +/* +get_pack_length(org_key_length); */ + length= length-get_pack_length(ref_length)+ + get_pack_length(new_pack_length); + return (int) length + s_temp->prev_length; + } + } + else + { + /* Next key wasn't a prefix of previous key */ + ref_length=0; + next_length_pack=0; + } + DBUG_PRINT("test",("length: %d next_key: %lx",length,next_key)); + + { + uint tmp_length; + key=(start+=ref_length); + if (key+n_length < key_end) /* Normalize length based */ + key_end=key+n_length; + if (sort_order) /* SerG */ + { + while (key < key_end && sort_order[*key] == + sort_order[*next_key]) + { + key++; next_key++; + } + } + else + { + while (key < key_end && *key == *next_key) + { + key++; next_key++; + } + } + if (!(tmp_length=(uint) (key-start))) + { /* Key can't be re-packed */ + s_temp->next_key_pos=0; + return length; + } + ref_length+=tmp_length; + n_length-=tmp_length; + length-=tmp_length+next_length_pack; /* We gained these chars */ + } + if (n_length == 0) + { + s_temp->n_ref_length=pack_marker; /* Same as prev key */ + } + else + { + s_temp->n_ref_length=ref_length | pack_marker; + length+= get_pack_length(n_length); + s_temp->n_length=n_length; + } + } + } + return length; +} + + +/* Length of key which is prefix compressed */ + +int +_mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) +{ + uint length,key_length,ref_length; + + s_temp->totlength=key_length=_mi_keylength(keyinfo,key)+nod_flag; + s_temp->key=key; + s_temp->prev_key=org_key; + if (prev_key) /* If not first key in block */ + { + /* pack key against previous key */ + /* + As keys may be identical when running a sort in myisamchk, we + have to guard against the case where keys may be identical + */ + uchar *end; + end=key+key_length; + for ( ; *key == *prev_key && key < end; key++,prev_key++) ; + s_temp->ref_length= ref_length=(uint) (key-s_temp->key); + length=key_length - ref_length + get_pack_length(ref_length); + } + else + { + /* No previous key */ + s_temp->ref_length=ref_length=0; + length=key_length+1; + } + if ((s_temp->next_key_pos=next_key)) /* If another key after */ + { + /* pack key against next key */ + uint next_length,next_length_pack; + get_key_pack_length(next_length,next_length_pack,next_key); + + /* If first key and next key is packed (only on delete) */ + if (!prev_key && org_key && next_length) + { + uchar *end; + for (key= s_temp->key, end=key+next_length ; + *key == *org_key && key < end; + key++,org_key++) ; + ref_length= (uint) (key - s_temp->key); + } + + if (next_length > ref_length) + { + /* We put a key with different case between two keys with the same prefix + Extend next key to have same prefix as + this key */ + s_temp->n_ref_length= ref_length; + s_temp->prev_length= next_length-ref_length; + s_temp->prev_key+= ref_length; + return (int) (length+ s_temp->prev_length - next_length_pack + + get_pack_length(ref_length)); + } + /* Check how many characters are identical to next key */ + key= s_temp->key+next_length; + while (*key++ == *next_key++) ; + if ((ref_length= (uint) (key - s_temp->key)-1) == next_length) + { + s_temp->next_key_pos=0; + return length; /* can't pack next key */ + } + s_temp->prev_length=0; + s_temp->n_ref_length=ref_length; + return (int) (length-(ref_length - next_length) - next_length_pack + + get_pack_length(ref_length)); + } + return (int) length; +} + + +/* +** store a key packed with _mi_calc_xxx_key_length in page-buffert +*/ + +/* store key without compression */ + +void _mi_store_static_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + memcpy((byte*) key_pos,(byte*) s_temp->key,(size_t) s_temp->totlength); +} + + +/* store variable length key with prefix compression */ + +#define store_pack_length(test,pos,length) { \ + if (test) { *((pos)++) = (uchar) (length); } else \ + { *((pos)++) = (uchar) ((length) >> 8); *((pos)++) = (uchar) (length); } } + + +void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + uint length; + uchar *start; + + start=key_pos; + + if (s_temp->ref_length) + { + /* Packed against previous key */ + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->ref_length); + /* If not same key after */ + if (s_temp->ref_length != s_temp->pack_marker) + store_key_length_inc(key_pos,s_temp->key_length); + } + else + { + /* Not packed against previous key */ + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->key_length); + } + bmove((byte*) key_pos,(byte*) s_temp->key, + (length=s_temp->totlength-(uint) (key_pos-start))); + + if (!s_temp->next_key_pos) /* No following key */ + return; + key_pos+=length; + + if (s_temp->prev_length) + { + /* Extend next key because new key didn't have same prefix as prev key */ + if (s_temp->part_of_prev_key) + { + store_pack_length(s_temp->pack_marker == 128,key_pos, + s_temp->part_of_prev_key); + store_key_length_inc(key_pos,s_temp->n_length); + } + else + { + s_temp->n_length+= s_temp->store_not_null; + store_pack_length(s_temp->pack_marker == 128,key_pos, + s_temp->n_length); + } + memcpy(key_pos, s_temp->prev_key, s_temp->prev_length); + } + else if (s_temp->n_ref_length) + { + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_ref_length); + if (s_temp->n_ref_length == s_temp->pack_marker) + return; /* Identical key */ + store_key_length(key_pos,s_temp->n_length); + } + else + { + s_temp->n_length+= s_temp->store_not_null; + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_length); + } +} + + +/* variable length key with prefix compression */ + +void _mi_store_bin_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + store_key_length_inc(key_pos,s_temp->ref_length); + memcpy((char*) key_pos,(char*) s_temp->key+s_temp->ref_length, + (size_t) s_temp->totlength-s_temp->ref_length); + + if (s_temp->next_key_pos) + { + key_pos+=(uint) (s_temp->totlength-s_temp->ref_length); + store_key_length_inc(key_pos,s_temp->n_ref_length); + if (s_temp->prev_length) /* If we must extend key */ + { + memcpy(key_pos,s_temp->prev_key,s_temp->prev_length); + } + } +} diff --git a/myisam/mi_static.c b/myisam/mi_static.c new file mode 100644 index 00000000000..4cc831e3f78 --- /dev/null +++ b/myisam/mi_static.c @@ -0,0 +1,56 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Static variables for pisam library. All definied here for easy making of + a shared library +*/ + +#ifndef _global_h +#include "myisamdef.h" +#endif + +LIST *myisam_open_list=0; +uchar NEAR myisam_file_magic[]= +{ (uchar) 254, (uchar) 254,'\007', '\001', }; +uchar NEAR myisam_pack_file_magic[]= +{ (uchar) 254, (uchar) 254,'\010', '\001', }; +my_string myisam_log_filename=(char*) "myisam.log"; +File myisam_log_file= -1; +uint myisam_quick_table_bits=9; +uint myisam_block_size=MI_KEY_BLOCK_LENGTH; /* Best by test */ +my_bool myisam_flush=0,myisam_delay_key_write=0; +#if defined(THREAD) && !defined(DONT_USE_RW_LOCKS) && defined(HAVE_PREAD) +my_bool myisam_concurrent_insert=1; +#else +my_bool myisam_concurrent_insert=0; +#endif + +/* read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ */ +/* Position is , == , >= , <= , > , < */ + +uint NEAR myisam_read_vec[]= +{ + SEARCH_FIND, SEARCH_FIND | SEARCH_BIGGER, SEARCH_FIND | SEARCH_SMALLER, + SEARCH_NO_FIND | SEARCH_BIGGER, SEARCH_NO_FIND | SEARCH_SMALLER, + SEARCH_FIND | SEARCH_PREFIX, SEARCH_LAST +}; + +uint NEAR myisam_readnext_vec[]= +{ + SEARCH_BIGGER, SEARCH_BIGGER, SEARCH_SMALLER, SEARCH_BIGGER, SEARCH_SMALLER, + SEARCH_BIGGER, SEARCH_SMALLER +}; diff --git a/myisam/mi_statrec.c b/myisam/mi_statrec.c new file mode 100644 index 00000000000..05ff40d8921 --- /dev/null +++ b/myisam/mi_statrec.c @@ -0,0 +1,302 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + /* Functions to handle fixed-length-records */ + +#include "myisamdef.h" + + +int _mi_write_static_record(MI_INFO *info, const byte *record) +{ + uchar temp[8]; /* max pointer length */ + + if (info->s->state.dellink != HA_OFFSET_ERROR) + { + my_off_t filepos=info->s->state.dellink; + info->rec_cache.seek_not_done=1; /* We have done a seek */ + VOID(my_seek(info->dfile,info->s->state.dellink+1,MY_SEEK_SET,MYF(0))); + + if (my_read(info->dfile,(char*) &temp[0],info->s->base.rec_reflength, + MYF(MY_NABP))) + goto err; + info->s->state.dellink= _mi_rec_pos(info->s,temp); + info->state->del--; + info->state->empty-=info->s->base.pack_reclength; + VOID(my_seek(info->dfile,filepos,MY_SEEK_SET,MYF(0))); + if (my_write(info->dfile, (char*) record, info->s->base.reclength, + MYF(MY_NABP))) + goto err; + } + else + { + if (info->state->data_file_length > info->s->base.max_data_file_length- + info->s->base.pack_reclength) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + return(2); + } + if (info->opt_flag & WRITE_CACHE_USED) + { /* Cash in use */ + if (my_b_write(&info->rec_cache, (byte*) record, + info->s->base.reclength)) + goto err; + if (info->s->base.pack_reclength != info->s->base.reclength) + { + uint length=info->s->base.pack_reclength - info->s->base.reclength; + bzero((char*) temp,length); + if (my_b_write(&info->rec_cache, (byte*) temp,length)) + goto err; + } + } + else + { + info->rec_cache.seek_not_done=1; /* We have done a seek */ + VOID(my_seek(info->dfile,info->state->data_file_length, + MY_SEEK_SET,MYF(0))); + if (my_write(info->dfile,(char*) record,info->s->base.reclength, + info->s->write_flag)) + goto err; + if (info->s->base.pack_reclength != info->s->base.reclength) + { + uint length=info->s->base.pack_reclength - info->s->base.reclength; + bzero((char*) temp,length); + if (my_write(info->dfile, (byte*) temp,length, info->s->write_flag)) + goto err; + } + } + info->state->data_file_length+=info->s->base.pack_reclength; + info->s->state.split++; + } + return 0; + err: + return 1; +} + +int _mi_update_static_record(MI_INFO *info, my_off_t pos, const byte *record) +{ + info->rec_cache.seek_not_done=1; /* We have done a seek */ + VOID(my_seek(info->dfile,pos,MY_SEEK_SET,MYF(0))); + return (my_write(info->dfile,(char*) record,info->s->base.reclength, + MYF(MY_NABP)) != 0); +} + + +int _mi_delete_static_record(MI_INFO *info) +{ + uchar temp[9]; /* 1+sizeof(uint32) */ + + info->state->del++; + info->state->empty+=info->s->base.pack_reclength; + temp[0]= '\0'; /* Mark that record is deleted */ + _mi_dpointer(info,temp+1,info->s->state.dellink); + info->s->state.dellink = info->lastpos; + info->rec_cache.seek_not_done=1; + VOID(my_seek(info->dfile,info->lastpos,MY_SEEK_SET,MYF(0))); + return (my_write(info->dfile,(byte*) temp, 1+info->s->rec_reflength, + MYF(MY_NABP)) != 0); +} + + +int _mi_cmp_static_record(register MI_INFO *info, register const byte *old) +{ + DBUG_ENTER("_mi_cmp_static_record"); + + /* We are going to do changes; dont let anybody disturb */ + dont_break(); /* Dont allow SIGHUP or SIGINT */ + + if (info->opt_flag & WRITE_CACHE_USED) + { + if (flush_io_cache(&info->rec_cache)) + { + DBUG_RETURN(-1); + } + info->rec_cache.seek_not_done=1; /* We have done a seek */ + } + + if ((info->opt_flag & READ_CHECK_USED)) + { /* If check isn't disabled */ + info->rec_cache.seek_not_done=1; /* We have done a seek */ + VOID(my_seek(info->dfile,info->lastpos,MY_SEEK_SET,MYF(0))); + if (my_read(info->dfile, (char*) info->rec_buff, info->s->base.reclength, + MYF(MY_NABP))) + DBUG_RETURN(-1); + if (memcmp((byte*) info->rec_buff, (byte*) old, + (uint) info->s->base.reclength)) + { + DBUG_DUMP("read",old,info->s->base.reclength); + DBUG_DUMP("disk",info->rec_buff,info->s->base.reclength); + my_errno=HA_ERR_RECORD_CHANGED; /* Record have changed */ + DBUG_RETURN(1); + } + } + DBUG_RETURN(0); +} + + +int _mi_cmp_static_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const byte *record, my_off_t pos) +{ + DBUG_ENTER("_mi_cmp_static_unique"); + + info->rec_cache.seek_not_done=1; /* We have done a seek */ + VOID(my_seek(info->dfile,pos,MY_SEEK_SET,MYF(0))); + if (my_read(info->dfile, (char*) info->rec_buff, info->s->base.reclength, + MYF(MY_NABP))) + DBUG_RETURN(-1); + DBUG_RETURN(mi_unique_comp(def, record, info->rec_buff, + def->null_are_equal)); +} + + + /* Read a fixed-length-record */ + /* Returns 0 if Ok. */ + /* 1 if record is deleted */ + /* MY_FILE_ERROR on read-error or locking-error */ + +int _mi_read_static_record(register MI_INFO *info, register my_off_t pos, + register byte *record) +{ + int error; + + if (pos != HA_OFFSET_ERROR) + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file <= pos && + flush_io_cache(&info->rec_cache)) + return(-1); + info->rec_cache.seek_not_done=1; /* We have done a seek */ + + error=my_pread(info->dfile,(char*) record,info->s->base.reclength, + pos,MYF(MY_NABP)) != 0; + if (info->s->r_locks == 0 && info->s->w_locks == 0) + VOID(_mi_writeinfo(info,0)); + if (! error) + { + if (!*record) + { + my_errno=HA_ERR_RECORD_DELETED; + return(1); /* Record is deleted */ + } + info->update|= HA_STATE_AKTIV; /* Record is read */ + return(0); + } + return(-1); /* Error on read */ + } + VOID(_mi_writeinfo(info,0)); /* No such record */ + return(-1); +} + + + +int _mi_read_rnd_static_record(MI_INFO *info, byte *buf, + register my_off_t filepos, + my_bool skipp_deleted_blocks) +{ + int locked,error,cache_read; + uint cache_length; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_static_record"); + + cache_read=0; + cache_length=0; + if (info->opt_flag & WRITE_CACHE_USED && + (info->rec_cache.pos_in_file <= filepos || skipp_deleted_blocks) && + flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + if (info->opt_flag & READ_CACHE_USED) + { /* Cache in use */ + if (filepos == my_b_tell(&info->rec_cache) && + (skipp_deleted_blocks || !filepos)) + { + cache_read=1; /* Read record using cache */ + cache_length=(uint) (info->rec_cache.rc_end - info->rec_cache.rc_pos); + } + else + info->rec_cache.seek_not_done=1; /* Filepos is changed */ + } + locked=0; + if (info->lock_type == F_UNLCK) + { + if (filepos >= info->state->data_file_length) + { /* Test if new records */ + if (_mi_readinfo(info,F_RDLCK,0)) + DBUG_RETURN(my_errno); + locked=1; + } + else + { /* We don't nead new info */ +#ifndef UNSAFE_LOCKING + if ((! cache_read || share->base.reclength > cache_length) && + share->r_locks == 0 && share->w_locks == 0) + { /* record not in cache */ + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) + DBUG_RETURN(my_errno); + locked=1; + } +#else + info->tmp_lock_type=F_RDLCK; +#endif + } + } + if (filepos >= info->state->data_file_length) + { + DBUG_PRINT("test",("filepos: %ld (%ld) records: %ld del: %ld", + filepos/share->base.reclength,filepos, + info->state->records, info->state->del)); + VOID(_mi_writeinfo(info,0)); + DBUG_RETURN(my_errno=HA_ERR_END_OF_FILE); + } + info->lastpos= filepos; + info->nextpos= filepos+share->base.pack_reclength; + + if (! cache_read) /* No cacheing */ + { + if ((error=_mi_read_static_record(info,filepos,buf))) + { + if (error > 0) + error=my_errno=HA_ERR_RECORD_DELETED; + else + error=my_errno; + } + DBUG_RETURN(error); + } + + /* Read record with cacheing */ + error=my_b_read(&info->rec_cache,(byte*) buf,share->base.reclength); + if (info->s->base.pack_reclength != info->s->base.reclength && !error) + { + char tmp[8]; /* Skill fill bytes */ + error=my_b_read(&info->rec_cache,(byte*) tmp, + info->s->base.pack_reclength - info->s->base.reclength); + } + if (locked) + VOID(_mi_writeinfo(info,0)); /* Unlock keyfile */ + if (!error) + { + if (!buf[0]) + { /* Record is removed */ + DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); + } + /* Found and may be updated */ + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + DBUG_RETURN(0); + } + /* my_errno should be set if rec_cache.error == -1 */ + if (info->rec_cache.error != -1 || my_errno == 0) + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(my_errno); /* Something wrong (EOF?) */ +} diff --git a/myisam/mi_test1.c b/myisam/mi_test1.c new file mode 100644 index 00000000000..59b9ceb266e --- /dev/null +++ b/myisam/mi_test1.c @@ -0,0 +1,640 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Testing of the basic functions of a MyISAM table */ + +#include "myisam.h" +#include <getopt.h> +#include <m_ctype.h> + + +#define MAX_REC_LENGTH 1024 + +static int rec_pointer_size=0,verbose=0,flags[50]; +static int key_field=FIELD_SKIPP_PRESPACE,extra_field=FIELD_SKIPP_ENDSPACE; +static int key_type=HA_KEYTYPE_NUM; +static int create_flag=0; + +static uint insert_count= 1000,update_count=1000,remove_count=1000; +static uint pack_keys=0,pack_seg=0,null_fields=0,key_length=6,skip_update=0; +static uint unique_key=HA_NOSAME,key_cacheing=0,opt_unique=0; +static uint silent; +static MI_COLUMNDEF recinfo[4]; +static MI_KEYDEF keyinfo[10]; +static MI_KEYSEG keyseg[10]; +static MI_KEYSEG uniqueseg[10]; + +static int run_test(const char *filename); +static void get_options(int argc, char *argv[]); +static void create_key(char *key,uint rownr); +static void create_record(char *record,uint rownr); +static void update_record(char *record); + +int main(int argc,char *argv[]) +{ + MY_INIT(argv[0]); + my_init(); + if (key_cacheing) + init_key_cache(IO_SIZE*16,(uint) IO_SIZE*4*10); + get_options(argc,argv); + + exit(run_test("test1")); +} + + +int run_test(const char *filename) +{ + MI_INFO *file; + int i,j,error,deleted,rec_length,uniques=0; + ha_rows found,row_count; + my_off_t pos; + char record[MAX_REC_LENGTH],key[MAX_REC_LENGTH],read_record[MAX_REC_LENGTH]; + MI_UNIQUEDEF uniquedef; + MI_CREATE_INFO create_info; + + bzero((char*) recinfo,sizeof(recinfo)); + + /* First define 2 columns */ + recinfo[0].type=FIELD_NORMAL; recinfo[0].length=1; /* For NULL bits */ + recinfo[1].type=key_field; + recinfo[1].length= (key_field == FIELD_BLOB ? 4+mi_portable_sizeof_char_ptr : + key_length); + if (key_field == FIELD_VARCHAR) + recinfo[1].length+=2; + recinfo[2].type=extra_field; + recinfo[2].length= (extra_field == FIELD_BLOB ? 4 + mi_portable_sizeof_char_ptr : 24); + if (extra_field == FIELD_VARCHAR) + recinfo[2].length+=2; + if (opt_unique) + { + recinfo[3].type=FIELD_CHECK; + recinfo[3].length=MI_UNIQUE_HASH_LENGTH; + } + rec_length=recinfo[0].length+recinfo[1].length+recinfo[2].length+ + recinfo[3].length; + + + /* Define a key over the first column */ + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=1; + keyinfo[0].seg[0].type= key_type; + keyinfo[0].seg[0].flag= pack_seg; + keyinfo[0].seg[0].start=1; + keyinfo[0].seg[0].length=key_length; + keyinfo[0].seg[0].null_bit= null_fields ? 2 : 0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].seg[0].language=MY_CHARSET_CURRENT; + if (pack_seg & HA_BLOB_PART) + { + keyinfo[0].seg[0].bit_start=4; /* Length of blob length */ + } + keyinfo[0].flag = (uint8) (pack_keys | unique_key); + + bzero((byte*) flags,sizeof(flags)); + if (opt_unique) + { + uint start; + uniques=1; + bzero((char*) &uniquedef,sizeof(uniquedef)); + bzero((char*) uniqueseg,sizeof(uniqueseg)); + uniquedef.seg=uniqueseg; + uniquedef.keysegs=2; + + /* Make a unique over all columns (except first NULL fields) */ + for (i=0, start=1 ; i < 2 ; i++) + { + uniqueseg[i].start=start; + start+=recinfo[i+1].length; + uniqueseg[i].length=recinfo[i+1].length; + uniqueseg[i].language=MY_CHARSET_CURRENT; + } + uniqueseg[0].type= key_type; + uniqueseg[0].null_bit= null_fields ? 2 : 0; + uniqueseg[1].type= HA_KEYTYPE_TEXT; + if (extra_field == FIELD_BLOB) + { + uniqueseg[1].length=0; /* The whole blob */ + uniqueseg[1].bit_start=4; /* long blob */ + uniqueseg[1].flag|= HA_BLOB_PART; + } + else if (extra_field == FIELD_VARCHAR) + uniqueseg[1].flag|= HA_VAR_LENGTH; + } + else + uniques=0; + + if (!silent) + printf("- Creating isam-file\n"); + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=(ulong) (rec_pointer_size ? + (1L << (rec_pointer_size*8))/40 : + 0); + if (mi_create(filename,1,keyinfo,3+opt_unique,recinfo, + uniques, &uniquedef, &create_info, + create_flag)) + goto err; + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + if (!silent) + printf("- Writing key:s\n"); + + my_errno=0; + row_count=deleted=0; + for (i=49 ; i>=1 ; i-=2 ) + { + if (insert_count-- == 0) { VOID(mi_close(file)) ; exit(0) ; } + j=i%25 +1; + create_record(record,j); + error=mi_write(file,record); + if (!error) + row_count++; + flags[j]=1; + if (verbose || error) + printf("J= %2d mi_write: %d errno: %d\n", j,error,my_errno); + } + + /* Insert 2 rows with null values */ + if (null_fields) + { + create_record(record,0); + error=mi_write(file,record); + if (!error) + row_count++; + if (verbose || error) + printf("J= NULL mi_write: %d errno: %d\n", error,my_errno); + error=mi_write(file,record); + if (!error) + row_count++; + if (verbose || error) + printf("J= NULL mi_write: %d errno: %d\n", error,my_errno); + flags[0]=2; + } + + if (!skip_update) + { + if (opt_unique) + { + if (!silent) + printf("- Checking unique constraint\n"); + create_record(record,j); + if (!mi_write(file,record) || my_errno != HA_ERR_FOUND_DUPP_UNIQUE) + { + printf("unique check failed\n"); + } + } + if (!silent) + printf("- Updating rows\n"); + + /* Update first last row to force extend of file */ + if (mi_rsame(file,read_record,-1)) + { + printf("Can't find last row with mi_rsame\n"); + } + else + { + memcpy(record,read_record,rec_length); + update_record(record); + if (mi_update(file,read_record,record)) + { + printf("Can't update last row: %.*s\n", + keyinfo[0].seg[0].length,read_record+1); + } + } + + /* Read through all rows and update them */ + pos=(my_off_t) 0; + found=0; + while ((error=mi_rrnd(file,read_record,pos)) == 0) + { + if (update_count-- == 0) { VOID(mi_close(file)) ; exit(0) ; } + memcpy(record,read_record,rec_length); + update_record(record); + if (mi_update(file,read_record,record)) + { + printf("Can't update row: %.*s, error: %d\n", + keyinfo[0].seg[0].length,record+1,my_errno); + } + found++; + pos=HA_OFFSET_ERROR; + } + if (found != row_count) + printf("Found %ld of %ld rows\n", found,row_count); + } + + if (!silent) + printf("- Reopening file\n"); + if (mi_close(file)) goto err; + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) goto err; + if (!skip_update) + { + if (!silent) + printf("- Removing keys\n"); + + for (i=0 ; i <= 10 ; i++) + { + /* testing */ + if (remove_count-- == 0) { VOID(mi_close(file)) ; exit(0) ; } + j=i*2; + if (!flags[j]) + continue; + create_key(key,j); + my_errno=0; + if ((error = mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT))) + { + if (verbose || (flags[j] >= 1 || + (error && my_errno != HA_ERR_KEY_NOT_FOUND))) + printf("key: '%.*s' mi_rkey: %3d errno: %3d\n", + (int) key_length,key+test(null_fields),error,my_errno); + } + else + { + error=mi_delete(file,read_record); + if (verbose || error) + printf("key: '%.*s' mi_delete: %3d errno: %3d\n", + (int) key_length, key+test(null_fields), error, my_errno); + if (! error) + { + deleted++; + flags[j]--; + } + } + } + } + if (!silent) + printf("- Reading rows with key\n"); + for (i=0 ; i <= 25 ; i++) + { + create_key(key,i); + my_errno=0; + error=mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT); + if (verbose || + (error == 0 && flags[i] == 0 && unique_key) || + (error && (flags[i] != 0 || my_errno != HA_ERR_KEY_NOT_FOUND))) + { + printf("key: '%.*s' mi_rkey: %3d errno: %3d record: %s\n", + (int) key_length,key+test(null_fields),error,my_errno,record+1); + } + } + + if (!silent) + printf("- Reading rows with position\n"); + for (i=1,found=0 ; i <= 30 ; i++) + { + my_errno=0; + if ((error=mi_rrnd(file,read_record,i == 1 ? 0L : HA_OFFSET_ERROR)) == -1) + { + if (found != row_count-deleted) + printf("Found only %ld of %ld rows\n",found,row_count-deleted); + break; + } + if (!error) + found++; + if (verbose || (error != 0 && error != HA_ERR_RECORD_DELETED && + error != HA_ERR_END_OF_FILE)) + { + printf("pos: %2d mi_rrnd: %3d errno: %3d record: %s\n", + i-1,error,my_errno,read_record+1); + } + } + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + + return (0); +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skipp warning */ +} + + +static void create_key_part(char *key,uint rownr) +{ + if (!unique_key) + rownr&=7; /* Some identical keys */ + if (keyinfo[0].seg[0].type == HA_KEYTYPE_NUM) + { + sprintf(key,"%*d",keyinfo[0].seg[0].length,rownr); + } + else if (keyinfo[0].seg[0].type == HA_KEYTYPE_VARTEXT) + { /* Alpha record */ + /* Create a key that may be easily packed */ + bfill(key,keyinfo[0].seg[0].length,rownr < 10 ? 'A' : 'B'); + sprintf(key+keyinfo[0].seg[0].length-2,"%-2d",rownr); + if ((rownr & 7) == 0) + { + /* Change the key to force a unpack of the next key */ + bfill(key+3,keyinfo[0].seg[0].length-4,rownr < 10 ? 'a' : 'b'); + } + } + else + { /* Alpha record */ + if (keyinfo[0].seg[0].flag & HA_SPACE_PACK) + sprintf(key,"%-*d",keyinfo[0].seg[0].length,rownr); + else + { + /* Create a key that may be easily packed */ + bfill(key,keyinfo[0].seg[0].length,rownr < 10 ? 'A' : 'B'); + sprintf(key+keyinfo[0].seg[0].length-2,"%-2d",rownr); + if ((rownr & 7) == 0) + { + /* Change the key to force a unpack of the next key */ + key[1]= (rownr < 10 ? 'a' : 'b'); + } + } + } +} + + +static void create_key(char *key,uint rownr) +{ + if (keyinfo[0].seg[0].null_bit) + { + if (rownr == 0) + { + key[0]=1; /* null key */ + key[1]=0; /* Fore easy print of key */ + return; + } + *key++=0; + } + if (keyinfo[0].seg[0].flag & (HA_BLOB_PART | HA_VAR_LENGTH)) + { + uint tmp; + create_key_part(key+2,rownr); + tmp=strlen(key+2); + int2store(key,tmp); + } + else + create_key_part(key,rownr); +} + + +static char blob_key[MAX_REC_LENGTH]; +static char blob_record[MAX_REC_LENGTH+20*20]; + + +static void create_record(char *record,uint rownr) +{ + char *pos; + bzero((char*) record,MAX_REC_LENGTH); + record[0]=1; /* delete marker */ + if (rownr == 0 && keyinfo[0].seg[0].null_bit) + record[0]|=keyinfo[0].seg[0].null_bit; /* Null key */ + + pos=record+1; + if (recinfo[1].type == FIELD_BLOB) + { + uint tmp; + char *ptr; + create_key_part(blob_key,rownr); + tmp=strlen(blob_key); + int4store(pos,tmp); + ptr=blob_key; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); + pos+=recinfo[1].length; + } + else if (recinfo[1].type == FIELD_VARCHAR) + { + uint tmp; + create_key_part(pos+2,rownr); + tmp=strlen(pos+2); + int2store(pos,tmp); + pos+=recinfo[1].length; + } + else + { + create_key_part(pos,rownr); + pos+=recinfo[1].length; + } + if (recinfo[2].type == FIELD_BLOB) + { + uint tmp; + char *ptr;; + sprintf(blob_record,"... row: %d", rownr); + strappend(blob_record,max(MAX_REC_LENGTH-rownr,10),' '); + tmp=strlen(blob_record); + int4store(pos,tmp); + ptr=blob_record; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); + } + else if (recinfo[2].type == FIELD_VARCHAR) + { + uint tmp; + sprintf(pos+2,"... row: %d", rownr); + tmp=strlen(pos+2); + int2store(pos,tmp); + } + else + { + sprintf(pos,"... row: %d", rownr); + strappend(pos,recinfo[2].length,' '); + } +} + +/* change row to test re-packing of rows and reallocation of keys */ + +static void update_record(char *record) +{ + char *pos=record+1; + if (recinfo[1].type == FIELD_BLOB) + { + char *column,*ptr; + int length; + length=uint4korr(pos); /* Long blob */ + memcpy_fixed(&column,pos+4,sizeof(char*)); + memcpy(blob_key,column,length); /* Move old key */ + ptr=blob_key; + memcpy_fixed(pos+4,&ptr,sizeof(char*)); /* Store pointer to new key */ + if (keyinfo[0].seg[0].type != HA_KEYTYPE_NUM) + casedn(blob_key,length); + pos+=recinfo[1].length; + } + else if (recinfo[1].type == FIELD_VARCHAR) + { + uint length=uint2korr(pos); + casedn(pos+2,length); + pos+=recinfo[1].length; + } + else + { + if (keyinfo[0].seg[0].type != HA_KEYTYPE_NUM) + casedn(pos,keyinfo[0].seg[0].length); + pos+=recinfo[1].length; + } + + if (recinfo[2].type == FIELD_BLOB) + { + char *column; + int length; + length=uint4korr(pos); + memcpy_fixed(&column,pos+4,sizeof(char*)); + memcpy(blob_record,column,length); + bfill(blob_record+length,20,'.'); /* Make it larger */ + length+=20; + int4store(pos,length); + column=blob_record; + memcpy_fixed(pos+4,&column,sizeof(char*)); + } + else if (recinfo[2].type == FIELD_VARCHAR) + { + /* Second field is longer than 10 characters */ + uint length=uint2korr(pos); + bfill(pos+2+length,recinfo[2].length-length-2,'.'); + length=recinfo[2].length-2; + int2store(pos,length); + } + else + { + bfill(pos+recinfo[2].length-10,10,'.'); + } +} + + +static struct option long_options[] = +{ + {"checksum", no_argument, 0, 'c'}, +#ifndef DBUG_OFF + {"debug", required_argument, 0, '#'}, +#endif + {"delete_rows", required_argument, 0, 'd'}, + {"help", no_argument, 0, '?'}, + {"insert_rows", required_argument, 0, 'i'}, + {"key_alpha", no_argument, 0, 'a'}, + {"key_binary_pack", no_argument, 0, 'B'}, + {"key_blob", required_argument, 0, 'b'}, + {"key_cache", no_argument, 0, 'K'}, + {"key_length", required_argument, 0, 'k'}, + {"key_multiple", no_argument, 0, 'm'}, + {"key_prefix_pack", no_argument, 0, 'P'}, + {"key_space_pack", no_argument, 0, 'p'}, + {"key_varchar", no_argument, 0, 'w'}, + {"null_fields", no_argument, 0, 'N'}, + {"row_fixed_size", no_argument, 0, 'S'}, + {"row_pointer_size", required_argument, 0, 'R'}, + {"silent", no_argument, 0, 's'}, + {"skip_update", no_argument, 0, 'U'}, + {"unique", no_argument, 0, 'C'}, + {"update_rows", required_argument, 0, 'u'}, + {"verbose", no_argument, 0, 'v'}, + {"version", no_argument, 0, 'V'}, + {0, 0, 0, 0} +}; + + +/* Read options */ + +static void get_options(int argc,char *argv[]) +{ + int c,option_index=0; + + while ((c=getopt_long(argc,argv,"abBcCd:i:k:KmPR:SspNu:UvVw#:", + long_options, &option_index)) != EOF) + { + switch(c) { + case 'a': + key_type= HA_KEYTYPE_TEXT; + break; + case 'c': + create_flag|= HA_CREATE_CHECKSUM; + break; + case 'C': + opt_unique=1; + break; + case 'R': /* Length of record pointer */ + rec_pointer_size=atoi(optarg); + if (rec_pointer_size > 3) + rec_pointer_size=0; + break; + case 'P': + pack_keys= HA_PACK_KEY; /* Use prefix compression */ + break; + case 'B': + pack_keys= HA_BINARY_PACK_KEY; /* Use binary compression */ + break; + case 'S': + if (key_field == FIELD_VARCHAR) + { + create_flag=0; /* Static sized varchar */ + } + else if (key_field != FIELD_BLOB) + { + key_field=FIELD_NORMAL; /* static-size record */ + extra_field=FIELD_NORMAL; + } + break; + case 'p': + pack_keys=HA_PACK_KEY; /* Use prefix + space packing */ + pack_seg=HA_SPACE_PACK; + key_type=HA_KEYTYPE_TEXT; + break; + case 'N': + null_fields=1; /* First key part may be null */ + break; + case 'v': /* verbose */ + verbose=1; + break; + case 'd': + remove_count=atoi(optarg); + break; + case 'i': + insert_count=atoi(optarg); + break; + case 'u': + update_count=atoi(optarg); + break; + case 'U': + skip_update=1; + break; + case 'm': + unique_key=0; + break; + case 'b': + key_field=FIELD_BLOB; /* blob key */ + extra_field= FIELD_BLOB; + pack_seg|= HA_BLOB_PART; + key_type= HA_KEYTYPE_VARTEXT; + break; + case 'k': + key_length=atoi(optarg); + if (key_length < 4 || key_length > MI_MAX_KEY_LENGTH) + { + fprintf(stderr,"Wrong key length\n"); + exit(1); + } + break; + case 's': + silent=1; + break; + case 'w': + key_field=FIELD_VARCHAR; /* varchar keys */ + extra_field= FIELD_VARCHAR; + key_type= HA_KEYTYPE_VARTEXT; + pack_seg|= HA_VAR_LENGTH; + create_flag|= HA_PACK_RECORD; + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + break; + case 'V': + printf("test1 Ver 1.0 \n"); + exit(0); + case '#': + DEBUGGER_ON; + DBUG_PUSH (optarg); + break; + } + } + return; +} /* get options */ diff --git a/myisam/mi_test2.c b/myisam/mi_test2.c new file mode 100644 index 00000000000..25bc1a4f844 --- /dev/null +++ b/myisam/mi_test2.c @@ -0,0 +1,973 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Test av isam-databas: stor test */ + +#ifndef USE_MY_FUNC /* We want to be able to dbug this !! */ +#define USE_MY_FUNC +#endif +#ifdef DBUG_OFF +#undef DBUG_OFF +#endif +#ifndef SAFEMALLOC +#define SAFEMALLOC +#endif +#include "myisamdef.h" +#include <m_ctype.h> + +#define STANDARD_LENGTH 37 +#define MYISAM_KEYS 6 +#define MAX_PARTS 4 +#if !defined(MSDOS) && !defined(labs) +#define labs(a) abs(a) +#endif + +static void get_options(int argc, char *argv[]); +static uint rnd(uint max_value); +static void fix_length(byte *record,uint length); +static void put_blob_in_record(char *blob_pos,char **blob_buffer); +static void copy_key(struct st_myisam_info *info,uint inx, + uchar *record,uchar *key); + +static int verbose=0,testflag=0, + first_key=0,async_io=0,key_cacheing=0,write_cacheing=0,locking=0, + rec_pointer_size=0,pack_fields=1,use_log=0,silent=0; +static int pack_seg=HA_SPACE_PACK,pack_type=HA_PACK_KEY,remove_count=-1, + create_flag=0; +static ulong key_cache_size=IO_SIZE*16; + +static uint keys=MYISAM_KEYS,recant=1000; +static uint use_blob=0; +static uint16 key1[1001],key3[5000]; +static char record[300],record2[300],key[100],key2[100], + read_record[300],read_record2[300],read_record3[300]; +static MI_KEYSEG glob_keyseg[MYISAM_KEYS][MAX_PARTS]; + + /* Test program */ + +int main(int argc, char **argv) +{ + uint i; + int j,n1,n2,n3,error,k; + uint write_count,update,dupp_keys,delete,start,length,blob_pos, + reclength,ant,found_parts; + my_off_t lastpos; + ha_rows range_records,records; + MI_INFO *file; + MI_KEYDEF keyinfo[10]; + MI_COLUMNDEF recinfo[10]; + MI_ISAMINFO info; + const char *filename; + char *blob_buffer; + MI_CREATE_INFO create_info; + MY_INIT(argv[0]); + + filename= "test2"; + get_options(argc,argv); + if (! async_io) + my_disable_async_io=1; + + reclength=STANDARD_LENGTH+60+(use_blob ? 8 : 0); + blob_pos=STANDARD_LENGTH+60; + keyinfo[0].seg= &glob_keyseg[0][0]; + keyinfo[0].seg[0].start=0; + keyinfo[0].seg[0].length=6; + keyinfo[0].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[0].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[0].seg[0].flag=(uint8) pack_seg; + keyinfo[0].seg[0].null_bit=0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].keysegs=1; + keyinfo[0].flag = pack_type; + keyinfo[1].seg= &glob_keyseg[1][0]; + keyinfo[1].seg[0].start=7; + keyinfo[1].seg[0].length=6; + keyinfo[1].seg[0].type=HA_KEYTYPE_BINARY; + keyinfo[1].seg[0].flag=0; + keyinfo[1].seg[0].null_bit=0; + keyinfo[1].seg[0].null_pos=0; + keyinfo[1].seg[1].start=0; /* two part key */ + keyinfo[1].seg[1].length=6; + keyinfo[1].seg[1].type=HA_KEYTYPE_NUM; + keyinfo[1].seg[1].flag=HA_REVERSE_SORT; + keyinfo[1].seg[1].null_bit=0; + keyinfo[1].seg[1].null_pos=0; + keyinfo[1].keysegs=2; + keyinfo[1].flag =0; + keyinfo[2].seg= &glob_keyseg[2][0]; + keyinfo[2].seg[0].start=12; + keyinfo[2].seg[0].length=8; + keyinfo[2].seg[0].type=HA_KEYTYPE_BINARY; + keyinfo[2].seg[0].flag=HA_REVERSE_SORT; + keyinfo[2].seg[0].null_bit=0; + keyinfo[2].seg[0].null_pos=0; + keyinfo[2].keysegs=1; + keyinfo[2].flag =HA_NOSAME; + keyinfo[3].seg= &glob_keyseg[3][0]; + keyinfo[3].seg[0].start=0; + keyinfo[3].seg[0].length=reclength-(use_blob ? 8 : 0); + keyinfo[3].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[3].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[3].seg[0].flag=(uint8) pack_seg; + keyinfo[3].seg[0].null_bit=0; + keyinfo[3].seg[0].null_pos=0; + keyinfo[3].keysegs=1; + keyinfo[3].flag = pack_type; + keyinfo[4].seg= &glob_keyseg[4][0]; + keyinfo[4].seg[0].start=0; + keyinfo[4].seg[0].length=5; + keyinfo[4].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[4].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[4].seg[0].flag=0; + keyinfo[4].seg[0].null_bit=0; + keyinfo[4].seg[0].null_pos=0; + keyinfo[4].keysegs=1; + keyinfo[4].flag = pack_type; + keyinfo[5].seg= &glob_keyseg[5][0]; + keyinfo[5].seg[0].start=0; + keyinfo[5].seg[0].length=4; + keyinfo[5].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[5].seg[0].language=MY_CHARSET_CURRENT; + keyinfo[5].seg[0].flag=pack_seg; + keyinfo[5].seg[0].null_bit=0; + keyinfo[5].seg[0].null_pos=0; + keyinfo[5].keysegs=1; + keyinfo[5].flag = pack_type; + + recinfo[0].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[0].length=7; + recinfo[0].null_bit=0; + recinfo[0].null_pos=0; + recinfo[1].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[1].length=5; + recinfo[1].null_bit=0; + recinfo[1].null_pos=0; + recinfo[2].type=pack_fields ? FIELD_SKIPP_PRESPACE : 0; + recinfo[2].length=9; + recinfo[2].null_bit=0; + recinfo[2].null_pos=0; + recinfo[3].type=FIELD_NORMAL; + recinfo[3].length=STANDARD_LENGTH-7-5-9-4; + recinfo[3].null_bit=0; + recinfo[3].null_pos=0; + recinfo[4].type=pack_fields ? FIELD_SKIPP_ZERO : 0; + recinfo[4].length=4; + recinfo[4].null_bit=0; + recinfo[4].null_pos=0; + recinfo[5].type=pack_fields ? FIELD_SKIPP_ENDSPACE : 0; + recinfo[5].length=60; + recinfo[5].null_bit=0; + recinfo[5].null_pos=0; + if (use_blob) + { + recinfo[6].type=FIELD_BLOB; + recinfo[6].length=4+mi_portable_sizeof_char_ptr; + recinfo[6].null_bit=0; + recinfo[6].null_pos=0; + } + + write_count=update=dupp_keys=delete=0; + blob_buffer=0; + + for (i=1000 ; i>0 ; i--) key1[i]=0; + for (i=4999 ; i>0 ; i--) key3[i]=0; + + if (!silent) + printf("- Creating isam-file\n"); + /* DBUG_PUSH(""); */ + /* my_delete(filename,MYF(0)); */ /* Remove old locks under gdb */ + file= 0; + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=(ha_rows) (rec_pointer_size ? + (1L << (rec_pointer_size*8))/ + reclength : 0); + create_info.reloc_rows=(ha_rows) 100; + if (mi_create(filename,keys,&keyinfo[first_key], + use_blob ? 7 : 6, &recinfo[0], + 0,(MI_UNIQUEDEF*) 0, + &create_info,create_flag)) + goto err; + if (use_log) + mi_log(1); + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + if (!silent) + printf("- Writing key:s\n"); + if (key_cacheing) + init_key_cache(key_cache_size,(uint) IO_SIZE*4*10); /* Use a small cache */ + if (locking) + mi_lock_database(file,F_WRLCK); + if (write_cacheing) + mi_extra(file,HA_EXTRA_WRITE_CACHE); + + for (i=0 ; i < recant ; i++) + { + n1=rnd(1000); n2=rnd(100); n3=rnd(5000); + sprintf(record,"%6d:%4d:%8d:Pos: %4d ",n1,n2,n3,write_count); + int4store(record+STANDARD_LENGTH-4,(long) i); + fix_length(record,(uint) STANDARD_LENGTH+rnd(60)); + put_blob_in_record(record+blob_pos,&blob_buffer); + DBUG_PRINT("test",("record: %d",i)); + + if (mi_write(file,record)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0) + { + printf("Error: %d in write at record: %d\n",my_errno,i); + goto err; + } + if (verbose) printf(" Double key: %d\n",n3); + } + else + { + if (key3[n3] == 1 && first_key <3 && first_key+keys >= 3) + { + printf("Error: Didn't get error when writing second key: '%8d'\n",n3); + goto err; + } + write_count++; key1[n1]++; key3[n3]=1; + } + + /* Check if we can find key without flushing database */ + if (i == recant/2) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (!j) + for (j=999 ; j>0 && key1[j] == 0 ; j--) ; + sprintf(key,"%6d",j); + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + { + printf("Test in loop: Can't find key: \"%s\"\n",key); + goto err; + } + } + } + if (testflag==1) goto end; + + if (write_cacheing) + if (mi_extra(file,HA_EXTRA_NO_CACHE)) + { + puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); + goto end; + } + + if (!silent) + printf("- Delete\n"); + for (i=0 ; i<recant/10 ; i++) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (j != 0) + { + sprintf(key,"%6d",j); + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + { + printf("can't find key1: \"%s\"\n",key); + goto err; + } + if (delete == (uint) remove_count) /* While testing */ + goto end; + if (mi_delete(file,read_record)) + { + printf("error: %d; can't delete record: \"%s\"\n", my_errno,read_record); + goto err; + } + delete++; + key1[atoi(read_record+keyinfo[0].seg[0].start)]--; + key3[atoi(read_record+keyinfo[2].seg[0].start)]=0; + } + else + puts("Warning: Skipping delete test because no dupplicate keys"); + } + if (testflag==2) goto end; + + if (!silent) + printf("- Update\n"); + for (i=0 ; i<recant/10 ; i++) + { + n1=rnd(1000); n2=rnd(100); n3=rnd(5000); + sprintf(record2,"%6d:%4d:%8d:XXX: %4d ",n1,n2,n3,update); + int4store(record2+STANDARD_LENGTH-4,(long) i); + fix_length(record2,(uint) STANDARD_LENGTH+rnd(60)); + + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (j != 0) + { + sprintf(key,"%6d",j); + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + { + printf("can't find key1: \"%s\"\n",key); + goto err; + } + if (use_blob) + { + if (i & 1) + put_blob_in_record(record+blob_pos,&blob_buffer); + else + bmove(record+blob_pos,read_record+blob_pos,8); + } + if (mi_update(file,read_record,record2)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0) + { + printf("error: %d; can't uppdate:\nFrom: \"%s\"\nTo: \"%s\"\n", + my_errno,read_record,record2); + goto err; + } + if (verbose) + printf("Double key when tryed to uppdate:\nFrom: \"%s\"\nTo: \"%s\"\n",record,record2); + } + else + { + key1[atoi(read_record+keyinfo[0].seg[0].start)]--; + key3[atoi(read_record+keyinfo[2].seg[0].start)]=0; + key1[n1]++; key3[n3]=1; + update++; + } + } + } + if (testflag==3) goto end; + + if (!silent) + printf("- Same key: first - next -> last - prev -> first\n"); + DBUG_PRINT("progpos",("first - next -> last - prev -> first")); + for (i=999, dupp_keys=j=0 ; i>0 ; i--) + { + if (key1[i] >dupp_keys) { dupp_keys=key1[i]; j=i; } + } + sprintf(key,"%6d",j); + if (verbose) printf(" Using key: \"%s\" Keys: %d\n",key,dupp_keys); + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) goto err; + if (mi_rsame(file,read_record2,-1)) goto err; + if (memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame didn't find same record\n"); + goto end; + } + info.recpos=mi_position(file); + if (mi_rfirst(file,read_record2,0) || + mi_rsame_with_pos(file,read_record2,0,info.recpos) || + memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame_with_pos didn't find same record\n"); + goto end; + } + { + int skr=mi_rnext(file,read_record2,0); + if ((skr && my_errno != HA_ERR_END_OF_FILE) || + mi_rprev(file,read_record2,-1) || + memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame_with_pos lost position\n"); + goto end; + } + } + ant=1; + start=keyinfo[0].seg[0].start; length=keyinfo[0].seg[0].length; + while (mi_rnext(file,read_record2,0) == 0 && + memcmp(read_record2+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("next: Found: %d keys of %d\n",ant,dupp_keys); + goto end; + } + ant=0; + while (mi_rprev(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("prev: Found: %d records of %d\n",ant,dupp_keys); + goto end; + } + + /* Check of mi_rnext_same */ + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) + goto err; + ant=1; + while (!mi_rnext_same(file,read_record3) && ant < dupp_keys+10) + ant++; + if (ant != dupp_keys || my_errno != HA_ERR_END_OF_FILE) + { + printf("mi_rnext_same: Found: %d records of %d\n",ant,dupp_keys); + goto end; + } + + if (!silent) + printf("- All keys: first - next -> last - prev -> first\n"); + DBUG_PRINT("progpos",("All keys: first - next -> last - prev -> first")); + ant=1; + if (mi_rfirst(file,read_record,0)) + { + printf("Can't find first record\n"); + goto end; + } + while ((error=mi_rnext(file,read_record3,0)) == 0 && ant < write_count+10) + ant++; + if (ant != write_count - delete || error != HA_ERR_END_OF_FILE) + { + printf("next: I found: %d records of %d (error: %d)\n", + ant, write_count - delete, error); + goto end; + } + if (mi_rlast(file,read_record2,0) || + bcmp(read_record2,read_record3,reclength)) + { + printf("Can't find last record\n"); + DBUG_DUMP("record2",(byte*) read_record2,reclength); + DBUG_DUMP("record3",(byte*) read_record3,reclength); + goto end; + } + ant=1; + while (mi_rprev(file,read_record3,0) == 0 && ant < write_count+10) + ant++; + if (ant != write_count - delete) + { + printf("prev: I found: %d records of %d\n",ant,write_count); + goto end; + } + if (bcmp(read_record,read_record3,reclength)) + { + printf("Can't find first record\n"); + goto end; + } + + if (!silent) + printf("- Test if: Read first - next - prev - prev - next == first\n"); + DBUG_PRINT("progpos",("- Read first - next - prev - prev - next == first")); + if (mi_rfirst(file,read_record,0) || + mi_rnext(file,read_record3,0) || + mi_rprev(file,read_record3,0) || + mi_rprev(file,read_record3,0) == 0 || + mi_rnext(file,read_record3,0)) + goto err; + if (bcmp(read_record,read_record3,reclength) != 0) + printf("Can't find first record\n"); + + if (!silent) + printf("- Test if: Read last - prev - next - next - prev == last\n"); + DBUG_PRINT("progpos",("Read last - prev - next - next - prev == last")); + if (mi_rlast(file,read_record2,0) || + mi_rprev(file,read_record3,0) || + mi_rnext(file,read_record3,0) || + mi_rnext(file,read_record3,0) == 0 || + mi_rprev(file,read_record3,0)) + goto err; + if (bcmp(read_record2,read_record3,reclength)) + printf("Can't find last record\n"); + + if (!silent) + puts("- Test read key-part"); + strmov(key2,key); + for(i=strlen(key2) ; i-- > 1 ;) + { + key2[i]=0; + + /* The following row is just to catch some bugs in the key code */ + bzero((char*) file->lastkey,file->s->base.max_key_length*2); + if (mi_rkey(file,read_record,0,key2,(uint) i,HA_READ_PREFIX)) + goto err; + if (bcmp(read_record+start,key,(uint) i)) + { + puts("Didn't find right record"); + goto end; + } + } + if (dupp_keys > 2) + { + if (!silent) + printf("- Read key (first) - next - delete - next -> last\n"); + DBUG_PRINT("progpos",("first - next - delete - next -> last")); + if (mi_rkey(file,read_record,0,key,0,HA_READ_KEY_EXACT)) goto err; + if (mi_rnext(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + delete++; + ant=1; + while (mi_rnext(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-1) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-1); + goto end; + } + } + if (dupp_keys>4) + { + if (!silent) + printf("- Read last of key - prev - delete - prev -> first\n"); + DBUG_PRINT("progpos",("last - prev - delete - prev -> first")); + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + delete++; + ant=1; + while (mi_rprev(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-2) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-2); + goto end; + } + } + if (dupp_keys > 6) + { + if (!silent) + printf("- Read first - delete - next -> last\n"); + DBUG_PRINT("progpos",("first - delete - next -> last")); + if (mi_rkey(file,read_record3,0,key,0,HA_READ_KEY_EXACT)) goto err; + if (mi_delete(file,read_record3)) goto err; + delete++; + ant=1; + if (mi_rnext(file,read_record,0)) + goto err; /* Skall finnas poster */ + while (mi_rnext(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-3) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-3); + goto end; + } + + if (!silent) + printf("- Read last - delete - prev -> first\n"); + DBUG_PRINT("progpos",("last - delete - prev -> first")); + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + delete++; + ant=0; + while (mi_rprev(file,read_record3,0) == 0 && + bcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-4) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-4); + goto end; + } + } + + if (!silent) + puts("- Test if: Read rrnd - same"); + DBUG_PRINT("progpos",("Read rrnd - same")); + for (i=0 ; i < write_count ; i++) + { + if (mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR) == 0) + break; + } + if (i == write_count) + goto err; + + bmove(read_record2,read_record,reclength); + for (i=min(2,keys) ; i-- > 0 ;) + { + if (mi_rsame(file,read_record2,(int) i)) goto err; + if (bcmp(read_record,read_record2,reclength) != 0) + { + printf("is_rsame didn't find same record\n"); + goto end; + } + } + if (!silent) + puts("- Test mi_records_in_range"); + mi_status(file,&info,HA_STATUS_VARIABLE); + for (i=0 ; i < info.keys ; i++) + { + if (mi_rfirst(file,read_record,(int) i) || + mi_rlast(file,read_record2,(int) i)) + goto err; + copy_key(file,(uint) i,(uchar*) read_record,(uchar*) key); + copy_key(file,(uint) i,(uchar*) read_record2,(uchar*) key2); + range_records=mi_records_in_range(file,(int) i,key,0,HA_READ_KEY_EXACT, + key2,0,HA_READ_AFTER_KEY); + if (range_records < info.records*8/10 || + range_records > info.records*12/10) + { + printf("mi_records_range returned %ld; Should be about %ld\n", + (long) range_records,(long) info.records); + goto end; + } + if (verbose) + { + printf("mi_records_range returned %ld; Exact is %ld (diff: %4.2g %%)\n", + (long) range_records, (long) info.records, + labs((long) range_records - (long) info.records)*100.0/ + info.records); + } + } + for (i=0 ; i < 5 ; i++) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + for (k=rnd(1000)+1 ; k>0 && key1[k] == 0 ; k--) ; + if (j != 0 && k != 0) + { + if (j > k) + swap(int,j,k); + sprintf(key,"%6d",j); + sprintf(key2,"%6d",k); + range_records=mi_records_in_range(file,0,key,0,HA_READ_AFTER_KEY, + key2,0,HA_READ_BEFORE_KEY); + records=0; + for (j++ ; j < k ; j++) + records+=key1[j]; + if ((long) range_records < (long) records*7/10-2 || + (long) range_records > (long) records*14/10+2) + { + printf("mi_records_range for key: %d returned %ld; Should be about %ld\n", + i, range_records, records); + goto end; + } + if (verbose && records) + { + printf("mi_records_range returned %ld; Exact is %ld (diff: %4.2g %%)\n", + range_records,records, + labs((long) range_records-(long) records)*100.0/records); + + } + } + } + + if (!silent) + printf("- mi_info\n"); + mi_status(file,&info,HA_STATUS_VARIABLE | HA_STATUS_CONST); + if (info.records != write_count-delete || info.deleted > delete + update + || info.keys != keys) + { + puts("Wrong info from mi_info"); + printf("Got: records: %ld delete: %ld i_keys: %d\n", + info.records,info.deleted,info.keys); + } + if (verbose) + { + char buff[80]; + get_date(buff,3,info.create_time); + printf("info: Created %s\n",buff); + get_date(buff,3,info.check_time); + printf("info: checked %s\n",buff); + get_date(buff,3,info.update_time); + printf("info: Modified %s\n",buff); + } + + mi_panic(HA_PANIC_WRITE); + mi_panic(HA_PANIC_READ); + if (mi_is_changed(file)) + puts("Warning: mi_is_changed reported that datafile was changed"); + + if (!silent) + printf("- mi_extra(CACHE) + mi_rrnd.... + mi_extra(NO_CACHE)\n"); + if (mi_extra(file,HA_EXTRA_RESET) || mi_extra(file,HA_EXTRA_CACHE)) + { + if (locking || (!use_blob && !pack_fields)) + { + puts("got error from mi_extra(HA_EXTRA_CACHE)"); + goto end; + } + } + ant=0; + while ((error=mi_rrnd(file,record,HA_OFFSET_ERROR)) != HA_ERR_END_OF_FILE && + ant < write_count + 10) + ant+= error ? 0 : 1; + if (ant != write_count-delete) + { + printf("rrnd with cache: I can only find: %d records of %d\n", + ant,write_count-delete); + goto end; + } + if (mi_extra(file,HA_EXTRA_NO_CACHE)) + { + puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); + goto end; + } + + ant=0; + mi_scan_init(file); + while ((error=mi_scan(file,record)) != HA_ERR_END_OF_FILE && + ant < write_count + 10) + ant+= error ? 0 : 1; + if (ant != write_count-delete) + { + printf("scan with cache: I can only find: %d records of %d\n", + ant,write_count-delete); + goto end; + } + + if (testflag == 4) goto end; + + if (!silent) + printf("- Removing keys\n"); + DBUG_PRINT("progpos",("Removing keys")); + lastpos = HA_OFFSET_ERROR; + /* DBUG_POP(); */ + mi_extra(file,HA_EXTRA_RESET); + found_parts=0; + while ((error=mi_rrnd(file,read_record,HA_OFFSET_ERROR)) != + HA_ERR_END_OF_FILE) + { + info.recpos=mi_position(file); + if (lastpos >= info.recpos && lastpos != HA_OFFSET_ERROR) + { + printf("mi_rrnd didn't advance filepointer; old: %ld, new: %ld\n", + (long) lastpos, (long) info.recpos); + goto err; + } + lastpos=info.recpos; + if (error == 0) + { + if (delete == (uint) remove_count) /* While testing */ + goto end; + if (mi_rsame(file,read_record,-1)) + { + printf("can't find record %lx\n",(long) info.recpos); + goto err; + } + if (use_blob) + { + ulong blob_length,pos; + uchar *ptr; + longget(blob_length,read_record+blob_pos+4); + ptr=(uchar*) blob_length; + longget(blob_length,read_record+blob_pos); + for (pos=0 ; pos < blob_length ; pos++) + { + if (ptr[pos] != (uchar) (blob_length+pos)) + { + printf("found blob with wrong info at %ld\n",(long) lastpos); + use_blob=0; + break; + } + } + } + if (mi_delete(file,read_record)) + { + printf("can't delete record: %6.6s, delete_count: %d\n", + read_record, delete); + goto err; + } + delete++; + } + else + found_parts++; + } + if (my_errno != HA_ERR_END_OF_FILE && my_errno != HA_ERR_RECORD_DELETED) + printf("error: %d from mi_rrnd\n",my_errno); + if (write_count != delete) + { + printf("Deleted only %d of %d records (%d parts)\n",delete,write_count, + found_parts); + goto err; + } +end: + if (mi_close(file)) + goto err; + mi_panic(HA_PANIC_CLOSE); /* Should close log */ + if (!silent) + { + printf("\nFollowing test have been made:\n"); + printf("Write records: %d\nUpdate records: %d\nSame-key-read: %d\nDelete records: %d\n", write_count,update,dupp_keys,delete); + if (rec_pointer_size) + printf("Record pointer size: %d\n",rec_pointer_size); + if (key_cacheing) + puts("Key cacheing used"); + if (write_cacheing) + puts("Write cacheing used"); + if (async_io && locking) + puts("Asyncron io with locking used"); + else if (locking) + puts("Locking used"); + if (use_blob) + puts("blobs used"); + printf("key cache status: \n\ +blocks used:%10lu\n\ +w_requests: %10lu\n\ +writes: %10lu\n\ +r_requests: %10lu\n\ +reads: %10lu\n", + _my_blocks_used,_my_cache_w_requests, _my_cache_write, + _my_cache_r_requests,_my_cache_read); + } + end_key_cache(); + if (blob_buffer) + my_free(blob_buffer,MYF(0)); + my_end(MY_CHECK_ERROR | MY_GIVE_INFO); + return(0); +err: + printf("got error: %d when using MyISAM-database\n",my_errno); + if (file) + VOID(mi_close(file)); + return(1); +} /* main */ + + + /* l{ser optioner */ + /* OBS! intierar endast DEBUG - ingen debuggning h{r ! */ + +static void get_options(int argc, char **argv) +{ + char *pos,*progname; + DEBUGGER_OFF; + + progname= argv[0]; + + while (--argc >0 && *(pos = *(++argv)) == '-' ) { + switch(*++pos) { + case 'B': + pack_type= HA_BINARY_PACK_KEY; + break; + case 'b': + use_blob=1; + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + if (*++pos) + key_cache_size=atol(pos); + break; + case 'W': /* Use write cacheing */ + write_cacheing=1; + if (*++pos) + my_default_record_cache_size=atoi(pos); + break; + case 'd': + remove_count= atoi(++pos); + break; + case 'i': + if (*++pos) + srand(atoi(pos)); + break; + case 'l': + use_log=1; + break; + case 'L': + locking=1; + break; + case 'A': /* use asyncron io */ + async_io=1; + if (*++pos) + my_default_record_cache_size=atoi(pos); + break; + case 'v': /* verbose */ + verbose=1; + break; + case 'm': /* records */ + recant=atoi(++pos); + break; + case 'f': + if ((first_key=atoi(++pos)) < 0 || first_key >= MYISAM_KEYS) + first_key=0; + break; + case 'k': + if ((keys=(uint) atoi(++pos)) < 1 || + keys > (uint) (MYISAM_KEYS-first_key)) + keys=MYISAM_KEYS-first_key; + break; + case 'P': + pack_type=0; /* Don't use DIFF_LENGTH */ + pack_seg=0; + break; + case 'R': /* Length of record pointer */ + rec_pointer_size=atoi(++pos); + if (rec_pointer_size > 7) + rec_pointer_size=0; + break; + case 'S': + pack_fields=0; /* Static-length-records */ + break; + case 's': + silent=1; + break; + case 't': + testflag=atoi(++pos); /* testmod */ + break; + case 'c': + create_flag|= HA_CREATE_CHECKSUM; + break; + case 'D': + create_flag|=HA_CREATE_DELAY_KEY_WRITE; + break; + case '?': + case 'I': + case 'V': + printf("%s Ver 1.1 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + printf("Usage: %s [-?AbBcDIKLPRSsVWltv] [-k#] [-f#] [-m#] [-t#]\n", + progname); + exit(0); + case '#': + DEBUGGER_ON; + DBUG_PUSH (++pos); + break; + default: + printf("Illegal option: '%c'\n",*pos); + break; + } + } + return; +} /* get options */ + + /* Get a random value 0 <= x <= n */ + +static uint rnd(uint max_value) +{ + return (uint) ((rand() & 32767)/32767.0*max_value); +} /* rnd */ + + + /* Create a variable length record */ + +static void fix_length(byte *rec, uint length) +{ + bmove(rec+STANDARD_LENGTH, + "0123456789012345678901234567890123456789012345678901234567890", + length-STANDARD_LENGTH); + strfill(rec+length,STANDARD_LENGTH+60-length,' '); +} /* fix_length */ + + + /* Put maybe a blob in record */ + +static void put_blob_in_record(char *blob_pos, char **blob_buffer) +{ + ulong i,length; + if (use_blob) + { + if (rnd(10) == 0) + { + if (! *blob_buffer && + !(*blob_buffer=my_malloc((uint) use_blob,MYF(MY_WME)))) + { + use_blob=0; + return; + } + length=rnd(use_blob); + for (i=0 ; i < length ; i++) + (*blob_buffer)[i]=(char) (length+i); + int4store(blob_pos,length); + memcpy_fixed(blob_pos+4,(char*) blob_buffer,sizeof(char*)); + } + else + { + int4store(blob_pos,0); + } + } + return; +} + + +static void copy_key(MI_INFO *info,uint inx,uchar *rec,uchar *key_buff) +{ + MI_KEYSEG *keyseg; + + for (keyseg=info->s->keyinfo[inx].seg ; keyseg->type ; keyseg++) + { + memcpy(key_buff,rec+keyseg->start,(size_t) keyseg->length); + key_buff+=keyseg->length; + } + return; +} diff --git a/myisam/mi_test3.c b/myisam/mi_test3.c new file mode 100644 index 00000000000..48325b2dcac --- /dev/null +++ b/myisam/mi_test3.c @@ -0,0 +1,485 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Test av locking */ + +#include "myisam.h" +#include <sys/types.h> +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif +#ifndef WEXITSTATUS +# define WEXITSTATUS(stat_val) ((unsigned)(stat_val) >> 8) +#endif +#ifndef WIFEXITED +# define WIFEXITED(stat_val) (((stat_val) & 255) == 0) +#endif + + +#if defined(HAVE_LRAND48) +#define rnd(X) (lrand48() % X) +#define rnd_init(X) srand48(X) +#else +#define rnd(X) (random() % X) +#define rnd_init(X) srandom(X) +#endif + + +const char *filename= "test3.MSI"; +uint tests=10,forks=10,key_cacheing=0,use_log=0; + +static void get_options(int argc, char *argv[]); +void start_test(int id); +int test_read(MI_INFO *,int),test_write(MI_INFO *,int,int), + test_update(MI_INFO *,int,int),test_rrnd(MI_INFO *,int); + +struct record { + char id[8]; + char nr[4]; + char text[10]; +} record; + + +int main(int argc,char **argv) +{ + int status,wait_ret; + uint i=0; + MI_KEYDEF keyinfo[10]; + MI_COLUMNDEF recinfo[10]; + MI_KEYSEG keyseg[10][2]; + MY_INIT(argv[0]); + get_options(argc,argv); + + bzero((char*) keyinfo,sizeof(keyinfo)); + bzero((char*) recinfo,sizeof(recinfo)); + keyinfo[0].seg= &keyseg[0][0]; + keyinfo[0].seg[0].start=0; + keyinfo[0].seg[0].length=8; + keyinfo[0].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[0].seg[0].flag=HA_SPACE_PACK; + keyinfo[0].keysegs=1; + keyinfo[0].flag = (uint8) HA_PACK_KEY; + keyinfo[1].seg= &keyseg[1][0]; + keyinfo[1].seg[0].start=8; + keyinfo[1].seg[0].length=4; /* Long is always 4 in myisam */ + keyinfo[1].seg[0].type=HA_KEYTYPE_LONG_INT; + keyinfo[1].seg[0].flag=0; + keyinfo[1].keysegs=1; + keyinfo[1].flag =HA_NOSAME; + + recinfo[0].type=0; + recinfo[0].length=sizeof(record.id); + recinfo[1].type=0; + recinfo[1].length=sizeof(record.nr); + recinfo[2].type=0; + recinfo[2].length=sizeof(record.text); + + puts("- Creating myisam-file"); + my_delete(filename,MYF(0)); /* Remove old locks under gdb */ + if (mi_create(filename,2,&keyinfo[0],2,&recinfo[0],0,(MI_UNIQUEDEF*) 0, + (MI_CREATE_INFO*) 0,0)) + exit(1); + + rnd_init(0); + printf("- Starting %d processes\n",forks); fflush(stdout); + for (i=0 ; i < forks; i++) + { + if (!fork()) + { + start_test(i+1); + sleep(1); + return 0; + } + VOID(rnd(1)); + } + + for (i=0 ; i < forks ; i++) + while ((wait_ret=wait(&status)) && wait_ret == -1); + return 0; +} + + +static void get_options(int argc, char **argv) +{ + char *pos,*progname; + DEBUGGER_OFF; + + progname= argv[0]; + + while (--argc >0 && *(pos = *(++argv)) == '-' ) { + switch(*++pos) { + case 'l': + use_log=1; + break; + case 'f': + forks=atoi(++pos); + break; + case 't': + tests=atoi(++pos); + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + break; + case 'A': /* All flags */ + use_log=key_cacheing=1; + break; + case '?': + case 'I': + case 'V': + printf("%s Ver 1.0 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + puts("Test av locking with threads\n"); + printf("Usage: %s [-?lKA] [-f#] [-t#]\n",progname); + exit(0); + case '#': + DEBUGGER_ON; + DBUG_PUSH (++pos); + break; + default: + printf("Illegal option: '%c'\n",*pos); + break; + } + } + return; +} + + +void start_test(int id) +{ + uint i; + int error,lock_type; + MI_ISAMINFO isam_info; + MI_INFO *file,*file1,*file2=0,*lock; + + if (use_log) + mi_log(1); + if (!(file1=mi_open(filename,O_RDWR,HA_OPEN_WAIT_IF_LOCKED)) || + !(file2=mi_open(filename,O_RDWR,HA_OPEN_WAIT_IF_LOCKED))) + { + fprintf(stderr,"Can't open isam-file: %s\n",filename); + exit(1); + } + if (key_cacheing && rnd(2) == 0) + init_key_cache(65536L,(uint) IO_SIZE*4*10); + printf("Process %d, pid: %d\n",id,getpid()); fflush(stdout); + + for (error=i=0 ; i < tests && !error; i++) + { + file= (rnd(2) == 1) ? file1 : file2; + lock=0 ; lock_type=0; + if (rnd(10) == 0) + { + if (mi_lock_database(lock=(rnd(2) ? file1 : file2), + lock_type=(rnd(2) == 0 ? F_RDLCK : F_WRLCK))) + { + fprintf(stderr,"%2d: start: Can't lock table %d\n",id,my_errno); + error=1; + break; + } + } + switch (rnd(4)) { + case 0: error=test_read(file,id); break; + case 1: error=test_rrnd(file,id); break; + case 2: error=test_write(file,id,lock_type); break; + case 3: error=test_update(file,id,lock_type); break; + } + if (lock) + mi_lock_database(lock,F_UNLCK); + } + if (!error) + { + mi_status(file1,&isam_info,HA_STATUS_VARIABLE); + printf("%2d: End of test. Records: %ld Deleted: %ld\n", + id,isam_info.records,isam_info.deleted); + fflush(stdout); + } + + mi_close(file1); + mi_close(file2); + if (use_log) + mi_log(0); + if (error) + { + printf("%2d: Aborted\n",id); fflush(stdout); + exit(1); + } +} + + +int test_read(MI_INFO *file,int id) +{ + uint i,lock,found,next,prev; + ulong find; + + lock=0; + if (rnd(2) == 0) + { + lock=1; + if (mi_lock_database(file,F_RDLCK)) + { + fprintf(stderr,"%2d: Can't lock table %d\n",id,my_errno); + return 1; + } + } + + found=next=prev=0; + for (i=0 ; i < 100 ; i++) + { + find=rnd(100000); + if (!mi_rkey(file,record.id,1,(byte*) &find, + sizeof(find),HA_READ_KEY_EXACT)) + found++; + else + { + if (my_errno != HA_ERR_KEY_NOT_FOUND) + { + fprintf(stderr,"%2d: Got error %d from read in read\n",id,my_errno); + return 1; + } + else if (!mi_rnext(file,record.id,1)) + next++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in read\n",id,my_errno); + return 1; + } + else if (!mi_rprev(file,record.id,1)) + prev++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in read\n", + id,my_errno); + return 1; + } + } + } + } + } + if (lock) + { + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + return 1; + } + } + printf("%2d: read: found: %5d next: %5d prev: %5d\n", + id,found,next,prev); + fflush(stdout); + return 0; +} + + +int test_rrnd(MI_INFO *file,int id) +{ + uint count,lock; + + lock=0; + if (rnd(2) == 0) + { + lock=1; + if (mi_lock_database(file,F_RDLCK)) + { + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + mi_close(file); + return 1; + } + if (rnd(2) == 0) + mi_extra(file,HA_EXTRA_CACHE); + } + + count=0; + if (mi_rrnd(file,record.id,0L)) + { + if (my_errno == HA_ERR_END_OF_FILE) + goto end; + fprintf(stderr,"%2d: Can't read first record (%d)\n",id,my_errno); + return 1; + } + for (count=1 ; !mi_rrnd(file,record.id,HA_OFFSET_ERROR) ;count++) ; + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rrnd\n",id,my_errno); + return 1; + } + +end: + if (lock) + { + mi_extra(file,HA_EXTRA_NO_CACHE); + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + exit(0); + } + } + printf("%2d: rrnd: %5d\n",id,count); fflush(stdout); + return 0; +} + + +int test_write(MI_INFO *file,int id,int lock_type) +{ + uint i,tries,count,lock; + + lock=0; + if (rnd(2) == 0 || lock_type == F_RDLCK) + { + lock=1; + if (mi_lock_database(file,F_WRLCK)) + { + if (lock_type == F_RDLCK && my_errno == EDEADLK) + { + printf("%2d: write: deadlock\n",id); fflush(stdout); + return 0; + } + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + mi_close(file); + return 1; + } + if (rnd(2) == 0) + mi_extra(file,HA_EXTRA_WRITE_CACHE); + } + + sprintf(record.id,"%7d",getpid()); + strmov(record.text,"Testing..."); + + tries=(uint) rnd(100)+10; + for (i=count=0 ; i < tries ; i++) + { + uint32 tmp=rnd(80000)+20000; + int4store(record.nr,tmp); + if (!mi_write(file,record.id)) + count++; + else + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY) + { + fprintf(stderr,"%2d: Got error %d (errno %d) from write\n",id,my_errno, + errno); + return 1; + } + } + } + if (lock) + { + mi_extra(file,HA_EXTRA_NO_CACHE); + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + exit(0); + } + } + printf("%2d: write: %5d\n",id,count); fflush(stdout); + return 0; +} + + +int test_update(MI_INFO *file,int id,int lock_type) +{ + uint i,lock,found,next,prev,update; + uint32 tmp; + char find[4]; + struct record new_record; + + lock=0; + if (rnd(2) == 0 || lock_type == F_RDLCK) + { + lock=1; + if (mi_lock_database(file,F_WRLCK)) + { + if (lock_type == F_RDLCK && my_errno == EDEADLK) + { + printf("%2d: write: deadlock\n",id); fflush(stdout); + return 0; + } + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + return 1; + } + } + bzero((char*) &new_record,sizeof(new_record)); + strmov(new_record.text,"Updated"); + + found=next=prev=update=0; + for (i=0 ; i < 100 ; i++) + { + tmp=rnd(100000); + int4store(find,tmp); + if (!mi_rkey(file,record.id,1,(byte*) find, + sizeof(find),HA_READ_KEY_EXACT)) + found++; + else + { + if (my_errno != HA_ERR_KEY_NOT_FOUND) + { + fprintf(stderr,"%2d: Got error %d from read in update\n",id,my_errno); + return 1; + } + else if (!mi_rnext(file,record.id,1)) + next++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in update\n", + id,my_errno); + return 1; + } + else if (!mi_rprev(file,record.id,1)) + prev++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in update\n", + id,my_errno); + return 1; + } + continue; + } + } + } + memcpy_fixed(new_record.id,record.id,sizeof(record.id)); + tmp=rnd(20000)+40000; + int4store(new_record.nr,tmp); + if (!mi_update(file,record.id,new_record.id)) + update++; + else + { + if (my_errno != HA_ERR_RECORD_CHANGED && + my_errno != HA_ERR_RECORD_DELETED && + my_errno != HA_ERR_FOUND_DUPP_KEY) + { + fprintf(stderr,"%2d: Got error %d from update\n",id,my_errno); + return 1; + } + } + } + if (lock) + { + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"Can't unlock table,id, error%d\n",my_errno); + return 1; + } + } + printf("%2d: update: %5d\n",id,update); fflush(stdout); + return 0; +} diff --git a/myisam/mi_test_all.res b/myisam/mi_test_all.res new file mode 100644 index 00000000000..d6b4703d702 --- /dev/null +++ b/myisam/mi_test_all.res @@ -0,0 +1,48 @@ +Maximum memory usage: 545477 bytes (533k) +Maximum memory usage: 545477 bytes (533k) +Maximum memory usage: 545369 bytes (533k) +mi_test2-i686 -s -L -K -R1 -m2000 ; Should give error 135 +Error: 135 in write at record: 1105 +got error: 135 when using MyISAM-database +Maximum memory usage: 29439 bytes (29k) +Maximum memory usage: 66541 bytes (65k) +Maximum memory usage: 5101 bytes (5k) +Maximum memory usage: 545488 bytes (533k) +Commands Used count Errors Recover errors +open 1 0 0 +write 50 0 0 +update 5 0 0 +delete 50 0 0 +close 1 0 0 +extra 6 0 0 +Total 113 0 0 +Maximum memory usage: 545744 bytes (533k) +Commands Used count Errors Recover errors +open 2 0 0 +write 100 0 0 +update 10 0 0 +delete 100 0 0 +close 2 0 0 +extra 12 0 0 +Total 226 0 0 +Maximum memory usage: 5101 bytes (5k) +1.12user 0.52system 0:01.71elapsed 95%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (214major+25minor)pagefaults 0swaps +Maximum memory usage: 21189 bytes (21k) +1.29user 0.55system 0:01.82elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (215major+29minor)pagefaults 0swaps +Maximum memory usage: 66541 bytes (65k) +1.10user 0.51system 0:01.66elapsed 96%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (216major+40minor)pagefaults 0swaps +Maximum memory usage: 82629 bytes (81k) +1.33user 0.24system 0:01.54elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (216major+44minor)pagefaults 0swaps +Maximum memory usage: 545477 bytes (533k) +1.31user 0.24system 0:01.55elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (216major+173minor)pagefaults 0swaps +Maximum memory usage: 545389 bytes (533k) +1.37user 0.18system 0:01.57elapsed 98%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (218major+180minor)pagefaults 0swaps +Maximum memory usage: 109165 bytes (107k) +1.42user 0.38system 0:01.78elapsed 101%CPU (0avgtext+0avgdata 0maxresident)k +0inputs+0outputs (218major+51minor)pagefaults 0swaps diff --git a/myisam/mi_test_all.sh b/myisam/mi_test_all.sh new file mode 100755 index 00000000000..dfa2e1d0fdd --- /dev/null +++ b/myisam/mi_test_all.sh @@ -0,0 +1,120 @@ +silent="-s" +suffix=$MACH +mi_test1$suffix $silent +myisamchk$suffix -se test1 +mi_test1$suffix $silent -N -S +myisamchk$suffix -se test1 +mi_test1$suffix $silent -P --checksum +myisamchk$suffix -se test1 +mi_test1$suffix $silent -P -N -S +myisamchk$suffix -se test1 +mi_test1$suffix $silent -B -N -R2 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -k 480 --unique +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -N -S -R1 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -S +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -S -N --unique +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -S -N --key_length=127 --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -S -N --key_length=128 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -S --key_length=480 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -B +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -B --key_length=64 --unique +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -B -k 480 --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -B -k 480 -N --unique --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -m +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -m -P --unique --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -m -P --key_length=480 --key_cache +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -m -p +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -w -S --unique +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -w --key_length=64 --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -w -N --key_length=480 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -w -S --key_length=480 --checksum +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -b -N +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -a -b --key_length=480 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent -p -B --key_length=480 +myisamchk$suffix -sm test1 + +mi_test1$suffix $silent --checksum +myisamchk$suffix -se test1 +myisamchk$suffix -rs test1 +myisamchk$suffix -se test1 +myisamchk$suffix -rqs test1 +myisamchk$suffix -se test1 + +# check of myisampack / myisamchk +myisampack$suffix --force -s test1 +myisamchk$suffix -es test1 +myisamchk$suffix -rqs test1 +myisamchk$suffix -es test1 +myisamchk$suffix -rs test1 +myisamchk$suffix -es test1 +myisamchk$suffix -rus test1 +myisamchk$suffix -es test1 + +mi_test1$suffix $silent --checksum -S +myisamchk$suffix -se test1 +myisamchk$suffix -ros test1 +myisamchk$suffix -rqs test1 +myisamchk$suffix -se test1 + +myisampack$suffix --force -s test1 +myisamchk$suffix -rqs test1 +myisamchk$suffix -es test1 +myisamchk$suffix -rus test1 +myisamchk$suffix -es test1 + +mi_test1$suffix $silent --checksum --unique +myisamchk$suffix -se test1 +mi_test1$suffix $silent --unique -S +myisamchk$suffix -se test1 + + +mi_test1$suffix $silent --key_multiple -N -S +myisamchk$suffix -sm test1 +mi_test1$suffix $silent --key_multiple -a -p --key_length=480 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent --key_multiple -a -B --key_length=480 +myisamchk$suffix -sm test1 +mi_test1$suffix $silent --key_multiple -P -S +myisamchk$suffix -sm test1 + +mi_test2$suffix $silent -L -K -W -P +mi_test2$suffix $silent -L -K -W -P -A +mi_test2$suffix $silent -L -K -W -P -S -R1 -m500 +echo "mi_test2$suffix $silent -L -K -R1 -m2000 ; Should give error 135" +mi_test2$suffix $silent -L -K -R1 -m2000 +mi_test2$suffix $silent -L -K -P -S -R3 -m50 -b1000000 +mi_test2$suffix $silent -L -B +mi_test2$suffix $silent -D -B -c +mi_test2$suffix $silent -L -K -W -P -m50 -l +myisamlog$suffix +mi_test2$suffix $silent -L -K -W -P -m50 -l -b100 +myisamlog$suffix +time mi_test2$suffix $silent +time mi_test2$suffix $silent -K -B +time mi_test2$suffix $silent -L -B +time mi_test2$suffix $silent -L -K -B +time mi_test2$suffix $silent -L -K -W -B +time mi_test2$suffix $silent -L -K -W -S -B +time mi_test2$suffix $silent -D -K -W -S -B diff --git a/myisam/mi_unique.c b/myisam/mi_unique.c new file mode 100644 index 00000000000..e598fbeedb4 --- /dev/null +++ b/myisam/mi_unique.c @@ -0,0 +1,188 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Functions to check if a row is unique */ + +#include "myisamdef.h" +#include <m_ctype.h> + +my_bool mi_check_unique(MI_INFO *info, MI_UNIQUEDEF *def, byte *record, + ha_checksum unique_hash, my_off_t disk_pos) +{ + my_off_t lastpos=info->lastpos; + MI_KEYDEF *key= &info->s->keyinfo[def->key]; + uchar *key_buff=info->lastkey+info->s->base.max_key_length; + DBUG_ENTER("mi_check_unique"); + + mi_unique_store(record+key->seg->start, unique_hash); + _mi_make_key(info,def->key,key_buff,record,0); + + if (_mi_search(info,info->s->keyinfo+def->key,key_buff,MI_UNIQUE_HASH_LENGTH, + SEARCH_FIND,info->s->state.key_root[def->key])) + { + info->page_changed=1; /* Can't optimize read next */ + info->lastpos= lastpos; + DBUG_RETURN(0); /* No matching rows */ + } + + for (;;) + { + if (info->lastpos != disk_pos && + !(*info->s->compare_unique)(info,def,record,info->lastpos)) + { + my_errno=HA_ERR_FOUND_DUPP_UNIQUE; + info->errkey= (int) def->key; + info->dupp_key_pos= info->lastpos; + info->page_changed=1; /* Can't optimize read next */ + info->lastpos=lastpos; + DBUG_PRINT("info",("Found duplicate")); + DBUG_RETURN(1); /* Found identical */ + } + if (_mi_search_next(info,info->s->keyinfo+def->key, info->lastkey, + MI_UNIQUE_HASH_LENGTH, SEARCH_BIGGER, + info->s->state.key_root[def->key]) || + bcmp(info->lastkey, key_buff, MI_UNIQUE_HASH_LENGTH)) + { + info->page_changed=1; /* Can't optimize read next */ + info->lastpos=lastpos; + DBUG_RETURN(0); /* end of tree */ + } + } +} + + +/* Calculate a hash for a row */ + +ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record) +{ + const byte *pos, *end; + ha_checksum crc=0; + MI_KEYSEG *keyseg; + + for (keyseg=def->seg ; keyseg < def->end ; keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + uint length=keyseg->length; + + if (keyseg->null_bit) + { + if (record[keyseg->null_pos] & keyseg->null_bit) + continue; + } + pos= record+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH) + { + uint tmp_length=uint2korr(pos); + pos+=2; /* Skip VARCHAR length */ + set_if_smaller(length,tmp_length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint tmp_length=_mi_calc_blob_length(keyseg->bit_start,pos); + memcpy_fixed((byte*) &pos,pos+keyseg->bit_start,sizeof(char*)); + if (!length || length > tmp_length) + length=tmp_length; /* The whole blob */ + } + end= pos+length; + if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT) + { + uchar *sort_order=keyseg->charset->sort_order; + while (pos != end) + crc=((crc << 8) + + (((uchar) sort_order[*(uchar*) pos++]))) + + (crc >> (8*sizeof(ha_checksum)-8)); + } + else + while (pos != end) + crc=((crc << 8) + + (((uchar) *(uchar*) pos++))) + + (crc >> (8*sizeof(ha_checksum)-8)); + } + return crc; +} + + /* + Returns 0 if both rows have equal unique value + */ + +int mi_unique_comp(MI_UNIQUEDEF *def, const byte *a, const byte *b, + my_bool null_are_equal) +{ + const byte *pos_a, *pos_b, *end; + MI_KEYSEG *keyseg; + + for (keyseg=def->seg ; keyseg < def->end ; keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + uint length=keyseg->length; + + /* If part is NULL it's regarded as different */ + if (keyseg->null_bit) + { + uint tmp; + if ((tmp=(a[keyseg->null_pos] & keyseg->null_bit)) != + (uint) (b[keyseg->null_pos] & keyseg->null_bit)) + return 1; + if (tmp) + { + if (!null_are_equal) + return 1; + continue; + } + } + pos_a= a+keyseg->start; + pos_b= b+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH) + { + uint tmp_length=uint2korr(pos_a); + if (tmp_length != uint2korr(pos_b)) + return 1; + pos_a+=2; /* Skip VARCHAR length */ + pos_b+=2; + set_if_smaller(length,tmp_length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + /* Only compare 'length' characters if length<> 0 */ + uint a_length= _mi_calc_blob_length(keyseg->bit_start,pos_a); + uint b_length= _mi_calc_blob_length(keyseg->bit_start,pos_b); + /* Check that a and b are of equal length */ + if (length && a_length > length) + a_length=length; + if (!length || length > b_length) + length=b_length; + if (length != a_length) + return 1; + /* Both strings are at least 'length' long */ + memcpy_fixed((byte*) &pos_a,pos_a+keyseg->bit_start,sizeof(char*)); + memcpy_fixed((byte*) &pos_b,pos_b+keyseg->bit_start,sizeof(char*)); + } + end= pos_a+length; + if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT) + { + uchar *sort_order=keyseg->charset->sort_order; + while (pos_a != end) + if (sort_order[*(uchar*) pos_a++] != + sort_order[*(uchar*) pos_b++]) + return 1; + } + else + while (pos_a != end) + if (*pos_a++ != *pos_b++) + return 1; + } + return 0; +} diff --git a/myisam/mi_update.c b/myisam/mi_update.c new file mode 100644 index 00000000000..185624f3878 --- /dev/null +++ b/myisam/mi_update.c @@ -0,0 +1,184 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Update an old row in a MyISAM table */ + +#include "fulltext.h" +#ifdef __WIN__ +#include <errno.h> +#endif + + +int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec) +{ + int flag,key_changed,save_errno; + reg3 my_off_t pos; + uint i; + uchar old_key[MI_MAX_KEY_BUFF],*new_key; + bool auto_key_changed=0; + ulonglong changed; + MYISAM_SHARE *share=info->s; + ha_checksum old_checksum; + DBUG_ENTER("mi_update"); + LINT_INIT(new_key); + LINT_INIT(changed); + LINT_INIT(old_checksum); + + if (!(info->update & HA_STATE_AKTIV)) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); + } + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (info->state->key_file_length >= share->base.margin_key_file_length) + { + DBUG_RETURN(my_errno=HA_ERR_INDEX_FILE_FULL); + } + pos=info->lastpos; + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + + if (share->calc_checksum) + old_checksum=info->checksum=(*share->calc_checksum)(info,oldrec); + if ((*share->compare_record)(info,oldrec)) + { + save_errno=my_errno; + goto err_end; /* Record has changed */ + } + + /* Calculate and check all unique constraints */ + key_changed=0; + for (i=0 ; i < share->state.header.uniques ; i++) + { + MI_UNIQUEDEF *def=share->uniqueinfo+i; + if (mi_unique_comp(def, newrec, oldrec,1) && + mi_check_unique(info, def, newrec, mi_unique_hash(def, newrec), + info->lastpos)) + { + save_errno=my_errno; + goto err_end; + } + } + if (_mi_mark_file_changed(info)) + { + save_errno=my_errno; + goto err_end; + } + + /* Check which keys changed from the original row */ + + new_key=info->lastkey2; + key_changed=HA_STATE_KEY_CHANGED; /* We changed current database */ + /* Remove key that didn't change */ + changed=0; + for (i=0 ; i < share->base.keys ; i++) + { + if (((ulonglong) 1 << i) & share->state.key_map) + { + /* The following code block is for text searching by SerG */ + if (share->keyinfo[i].flag & HA_FULLTEXT ) + { + if(_mi_ft_cmp(info,i,oldrec, newrec)) + { + if ((int) i == info->lastinx) + key_changed|=HA_STATE_WRITTEN; + changed|=((ulonglong) 1 << i); + if (_mi_ft_del(info,i,(char*) old_key,oldrec,pos)) + goto err; + if (_mi_ft_add(info,i,(char*) new_key,newrec,pos)) + goto err; + } + } + else + { + uint new_length=_mi_make_key(info,i,new_key,newrec,pos); + uint old_length=_mi_make_key(info,i,old_key,oldrec,pos); + if (new_length != old_length || + memcmp((byte*) old_key,(byte*) new_key,new_length)) + { + if ((int) i == info->lastinx) + key_changed|=HA_STATE_WRITTEN; /* Mark that keyfile changed */ + changed|=((ulonglong) 1 << i); + share->keyinfo[i].version++; + if (_mi_ck_delete(info,i,old_key,old_length)) goto err; + if (_mi_ck_write(info,i,new_key,new_length)) goto err; + if (share->base.auto_key == i+1) + auto_key_changed=1; + } + } + } + } + + if (share->calc_checksum) + info->checksum=(*share->calc_checksum)(info,newrec); + if ((*share->update_record)(info,pos,newrec)) + goto err; + if (auto_key_changed) + update_auto_increment(info,newrec); + if (share->calc_checksum) + share->state.checksum+=(info->checksum - old_checksum); + + info->update= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED | HA_STATE_AKTIV | + key_changed); + myisam_log_record(MI_LOG_UPDATE,info,newrec,info->lastpos,0); + VOID(_mi_writeinfo(info,key_changed ? WRITEINFO_UPDATE_KEYFILE : 0)); + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(0); + +err: + DBUG_PRINT("error",("key: %d errno: %d",i,my_errno)); + save_errno=my_errno; + if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL) + { + info->errkey= (int) i; + flag=0; + do + { + if (((ulonglong) 1 << i) & changed) + { + /* The following code block is for text searching by SerG */ + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + if ((flag++ && _mi_ft_del(info,i,(char*) new_key,newrec,pos)) || + _mi_ft_add(info,i,(char*) old_key,oldrec,pos)) + break; + } + else + { + uint new_length=_mi_make_key(info,i,new_key,newrec,pos); + uint old_length= _mi_make_key(info,i,old_key,oldrec,pos); + if ((flag++ && _mi_ck_delete(info,i,new_key,new_length)) || + _mi_ck_write(info,i,old_key,old_length)) + break; + } + } + } while (i-- != 0); + } + else + mi_mark_crashed(info); + info->update= (HA_STATE_CHANGED | HA_STATE_AKTIV | HA_STATE_ROW_CHANGED | + key_changed); + + err_end: + myisam_log_record(MI_LOG_UPDATE,info,newrec,info->lastpos,my_errno); + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + allow_break(); /* Allow SIGHUP & SIGINT */ + if (save_errno == HA_ERR_KEY_NOT_FOUND) + save_errno=HA_ERR_CRASHED; + DBUG_RETURN(my_errno=save_errno); +} /* mi_update */ diff --git a/myisam/mi_write.c b/myisam/mi_write.c new file mode 100644 index 00000000000..f31e43e52ab --- /dev/null +++ b/myisam/mi_write.c @@ -0,0 +1,684 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Write a row to a MyISAM table */ + +#include "fulltext.h" +#ifdef __WIN__ +#include <errno.h> +#endif + +#define MAX_POINTER_LENGTH 8 + + /* Functions declared in this file */ + +static int w_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uint key_length, my_off_t pos, uchar *father_buff, + uchar *father_keypos, my_off_t father_page, + my_bool insert_last); +static int _mi_balance_page(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uchar *curr_buff,uchar *father_buff, + uchar *father_keypos,my_off_t father_page); +static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key); + + + /* Write new record to database */ + +int mi_write(MI_INFO *info, byte *record) +{ + uint i; + int save_errno; + my_off_t filepos; + uchar *buff; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_write"); + DBUG_PRINT("enter",("isam: %d data: %d",info->s->kfile,info->dfile)); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + dont_break(); /* Dont allow SIGHUP or SIGINT */ +#if !defined(NO_LOCKING) && defined(USE_RECORD_LOCK) + if (!info->locked && my_lock(info->dfile,F_WRLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) + goto err; +#endif + filepos= ((share->state.dellink != HA_OFFSET_ERROR) ? + share->state.dellink : + info->state->data_file_length); + + if (share->base.reloc == (ha_rows) 1 && + share->base.records == (ha_rows) 1 && + info->state->records == (ha_rows) 1) + { /* System file */ + my_errno=HA_ERR_RECORD_FILE_FULL; + goto err2; + } + if (info->state->key_file_length >= share->base.margin_key_file_length) + { + my_errno=HA_ERR_INDEX_FILE_FULL; + goto err2; + } + if (_mi_mark_file_changed(info)) + goto err2; + + /* Calculate and check all unique constraints */ + for (i=0 ; i < share->state.header.uniques ; i++) + { + if (mi_check_unique(info,share->uniqueinfo+i,record, + mi_unique_hash(share->uniqueinfo+i,record), + HA_OFFSET_ERROR)) + goto err2; + } + + /* Write all keys to indextree */ + + buff=info->lastkey2; + for (i=0 ; i < share->base.keys ; i++) + { + if (((ulonglong) 1 << i) & share->state.key_map) + { + if (share->concurrent_insert) + { + rw_wrlock(&share->key_root_lock[i]); + share->keyinfo[i].version++; + } + if (share->keyinfo[i].flag & HA_FULLTEXT ) /* SerG */ + { /* SerG */ + if (_mi_ft_add(info,i,(char*) buff,record,filepos)) /* SerG */ + { /* SerG */ + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + DBUG_PRINT("error",("Got error: %d on write",my_errno)); /* SerG */ + goto err; /* SerG */ + } /* SerG */ + } /* SerG */ + else /* SerG */ + { + uint key_length=_mi_make_key(info,i,buff,record,filepos); + if (_mi_ck_write(info,i,buff,key_length)) + { + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + DBUG_PRINT("error",("Got error: %d on write",my_errno)); + goto err; + } + } + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + } + } + if (share->calc_checksum) + info->checksum=(*share->calc_checksum)(info,record); + if (!(info->opt_flag & OPT_NO_ROWS)) + { + if ((*share->write_record)(info,record)) + goto err; + share->state.checksum+=info->checksum; + } + if (share->base.auto_key) + update_auto_increment(info,record); + info->update= (HA_STATE_CHANGED | HA_STATE_AKTIV | HA_STATE_WRITTEN | + HA_STATE_ROW_CHANGED); + info->state->records++; + info->lastpos=filepos; + myisam_log_record(MI_LOG_WRITE,info,record,filepos,0); + VOID(_mi_writeinfo(info, WRITEINFO_UPDATE_KEYFILE)); + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(0); + +err: + save_errno=my_errno; + if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL) + { + info->errkey= (int) i; + while ( i-- > 0) + { + if (((ulonglong) 1 << i) & share->state.key_map) + { + if (share->concurrent_insert) + rw_wrlock(&share->key_root_lock[i]); + /* The following code block is for text searching by SerG */ + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + if (_mi_ft_del(info,i,(char*) buff,record,filepos)) + { + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + break; + } + } + else + { + uint key_length=_mi_make_key(info,i,buff,record,filepos); + if (_mi_ck_delete(info,i,buff,key_length)) + { + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + break; + } + } + if (share->concurrent_insert) + rw_unlock(&share->key_root_lock[i]); + } + } + } + else + mi_mark_crashed(info); + info->update= (HA_STATE_CHANGED | HA_STATE_WRITTEN | HA_STATE_ROW_CHANGED); + my_errno=save_errno; +err2: + save_errno=my_errno; + myisam_log_record(MI_LOG_WRITE,info,record,filepos,my_errno); + VOID(_mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); + allow_break(); /* Allow SIGHUP & SIGINT */ + DBUG_RETURN(my_errno=save_errno); +} /* mi_write */ + + + /* Write one key to btree */ + +int _mi_ck_write(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + DBUG_ENTER("_mi_ck_write"); + + if (info->s->state.key_root[keynr] == HA_OFFSET_ERROR || + (error=w_search(info,info->s->keyinfo+keynr,key, key_length, + info->s->state.key_root[keynr], (uchar *) 0, (uchar*) 0, + (my_off_t) 0, 1)) > 0) + error=_mi_enlarge_root(info,keynr,key); + DBUG_RETURN(error); +} /* _mi_ck_write */ + + + /* Make a new root with key as only pointer */ + +int _mi_enlarge_root(register MI_INFO *info, uint keynr, uchar *key) +{ + uint t_length,nod_flag; + reg2 MI_KEYDEF *keyinfo; + MI_KEY_PARAM s_temp; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_enlarge_root"); + + nod_flag= (share->state.key_root[keynr] != HA_OFFSET_ERROR) ? + share->base.key_reflength : 0; + _mi_kpointer(info,info->buff+2,share->state.key_root[keynr]); /* if nod */ + keyinfo=share->keyinfo+keynr; + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,(uchar*) 0, + (uchar*) 0, (uchar*) 0, key,&s_temp); + mi_putint(info->buff,t_length+2+nod_flag,nod_flag); + (*keyinfo->store_key)(keyinfo,info->buff+2+nod_flag,&s_temp); + info->buff_used=info->page_changed=1; /* info->buff is used */ + if ((share->state.key_root[keynr]= _mi_new(info,keyinfo)) == + HA_OFFSET_ERROR || + _mi_write_keypage(info,keyinfo,share->state.key_root[keynr],info->buff)) + DBUG_RETURN(-1); + DBUG_RETURN(0); +} /* _mi_enlarge_root */ + + + /* + Search after a position for a key and store it there + Returns -1 = error + 0 = ok + 1 = key should be stored in higher tree + */ + +static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_length, my_off_t page, + uchar *father_buff, + uchar *father_keypos, my_off_t father_page, + my_bool insert_last) +{ + int error,flag; + uint comp_flag,nod_flag; + uchar *temp_buff,*keypos; + uchar keybuff[MI_MAX_KEY_BUFF]; + my_bool was_last_key; + my_off_t next_page; + DBUG_ENTER("w_search"); + DBUG_PRINT("enter",("page: %ld",page)); + + if (keyinfo->flag & HA_SORT_ALLOWS_SAME) + comp_flag=SEARCH_BIGGER; /* Put after same key */ + else if (keyinfo->flag & HA_NOSAME) + comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* No dupplicates */ + else + comp_flag=SEARCH_SAME; /* Keys in rec-pos order */ + + if (!(temp_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + MI_MAX_KEY_BUFF*2))) + DBUG_RETURN(-1); + if (!_mi_fetch_keypage(info,keyinfo,page,temp_buff,0)) + goto err; + + flag=(*keyinfo->bin_search)(info,keyinfo,temp_buff,key,key_length,comp_flag, + &keypos, keybuff, &was_last_key); + nod_flag=mi_test_if_nod(temp_buff); + if (flag == 0) + { + uint tmp_key_length; + my_errno=HA_ERR_FOUND_DUPP_KEY; + /* get position to record with duplicated key */ + tmp_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,keybuff); + if (tmp_key_length) + info->dupp_key_pos=_mi_dpos(info,0,keybuff+tmp_key_length); + else + info->dupp_key_pos= HA_OFFSET_ERROR; + my_afree((byte*) temp_buff); + DBUG_RETURN(-1); + } + if (flag == MI_FOUND_WRONG_KEY) + DBUG_RETURN(-1); + if (!was_last_key) + insert_last=0; + next_page=_mi_kpos(nod_flag,keypos); + if (next_page == HA_OFFSET_ERROR || + (error=w_search(info,keyinfo,key,key_length,next_page, + temp_buff, keypos, page, insert_last)) >0) + { + error=_mi_insert(info,keyinfo,key,temp_buff,keypos,keybuff,father_buff, + father_keypos,father_page, insert_last); + if (_mi_write_keypage(info,keyinfo,page,temp_buff)) + goto err; + } + my_afree((byte*) temp_buff); + DBUG_RETURN(error); +err: + my_afree((byte*) temp_buff); + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1); +} /* w_search */ + + + /* Insert new key at right of key_pos */ + /* Returns 2 if key contains key to upper level */ + +int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uchar *anc_buff, uchar *key_pos, uchar *key_buff, + uchar *father_buff, uchar *father_key_pos, my_off_t father_page, + my_bool insert_last) + +{ + uint a_length,nod_flag; + int t_length; + uchar *endpos, *prev_key; + MI_KEY_PARAM s_temp; + DBUG_ENTER("_mi_insert"); + DBUG_PRINT("enter",("key_pos: %lx",key_pos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,USE_WHOLE_KEY);); + + nod_flag=mi_test_if_nod(anc_buff); + a_length=mi_getint(anc_buff); + endpos= anc_buff+ a_length; + prev_key=(key_pos == anc_buff+2+nod_flag ? (uchar*) 0 : key_buff); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (key_pos == endpos ? (uchar*) 0 : key_pos), + prev_key, prev_key, + key,&s_temp); +#ifndef DBUG_OFF + if (key_pos != anc_buff+2+nod_flag && (keyinfo->flag & + (HA_BINARY_PACK_KEY | HA_PACK_KEY))) + DBUG_DUMP("prev_key",(byte*) key_buff,_mi_keylength(keyinfo,key_buff)); + if (keyinfo->flag & HA_PACK_KEY) + { + DBUG_PRINT("test",("t_length: %d ref_len: %d", + t_length,s_temp.ref_length)); + DBUG_PRINT("test",("n_ref_len: %d n_length: %d key: %lx", + s_temp.n_ref_length,s_temp.n_length,s_temp.key)); + } +#endif + if (t_length > 0) + { + if (t_length >= keyinfo->maxlength*2+MAX_POINTER_LENGTH) + { + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(-1); + } + bmove_upp((byte*) endpos+t_length,(byte*) endpos,(uint) (endpos-key_pos)); + } + else + { + if (-t_length >= keyinfo->maxlength*2+MAX_POINTER_LENGTH) + { + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(-1); + } + bmove(key_pos,key_pos-t_length,(uint) (endpos-key_pos)+t_length); + } + (*keyinfo->store_key)(keyinfo,key_pos,&s_temp); + a_length+=t_length; + mi_putint(anc_buff,a_length,nod_flag); + if (a_length <= keyinfo->block_length) + DBUG_RETURN(0); /* There is room on page */ + + /* Page is full */ + if (nod_flag) + insert_last=0; + if (!(keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY)) && + father_buff && !insert_last) + DBUG_RETURN(_mi_balance_page(info,keyinfo,key,anc_buff,father_buff, + father_key_pos,father_page)); + DBUG_RETURN(_mi_split_page(info,keyinfo,key,anc_buff,key_buff, insert_last)); +} /* _mi_insert */ + + + /* split a full page in two and assign emerging item to key */ + +int _mi_split_page(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uchar *buff, uchar *key_buff, + my_bool insert_last_key) +{ + uint length,a_length,key_ref_length,t_length,nod_flag,key_length; + uchar *key_pos,*pos, *after_key; + my_off_t new_pos; + MI_KEY_PARAM s_temp; + DBUG_ENTER("mi_split_page"); + DBUG_DUMP("buff",(byte*) buff,mi_getint(buff)); + + if (info->s->keyinfo+info->lastinx == keyinfo) + info->page_changed=1; /* Info->buff is used */ + info->buff_used=1; + nod_flag=mi_test_if_nod(buff); + key_ref_length=2+nod_flag; + if (insert_last_key) + key_pos=_mi_find_last_pos(keyinfo,buff,key_buff, &key_length, &after_key); + else + key_pos=_mi_find_half_pos(nod_flag,keyinfo,buff,key_buff, &key_length, + &after_key); + if (!key_pos) + DBUG_RETURN(-1); + length=(uint) (key_pos-buff); + a_length=mi_getint(buff); + mi_putint(buff,length,nod_flag); + + key_pos=after_key; + if (nod_flag) + { + DBUG_PRINT("test",("Splitting nod")); + pos=key_pos-nod_flag; + memcpy((byte*) info->buff+2,(byte*) pos,(size_t) nod_flag); + } + + /* Move middle item to key and pointer to new page */ + if ((new_pos=_mi_new(info,keyinfo)) == HA_OFFSET_ERROR) + DBUG_RETURN(-1); + _mi_kpointer(info,_mi_move_key(keyinfo,key,key_buff),new_pos); + + /* Store new page */ + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&key_pos,key_buff)) + DBUG_RETURN(-1); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,(uchar *) 0, + (uchar*) 0, (uchar*) 0, + key_buff, &s_temp); + length=(uint) ((buff+a_length)-key_pos); + memcpy((byte*) info->buff+key_ref_length+t_length,(byte*) key_pos, + (size_t) length); + (*keyinfo->store_key)(keyinfo,info->buff+key_ref_length,&s_temp); + mi_putint(info->buff,length+t_length+key_ref_length,nod_flag); + + if (_mi_write_keypage(info,keyinfo,new_pos,info->buff)) + DBUG_RETURN(-1); + DBUG_DUMP("key",(byte*) key,_mi_keylength(keyinfo,key)); + DBUG_RETURN(2); /* Middle key up */ +} /* _mi_split_page */ + + + /* + Calculate how to much to move to split a page in two + Returns pointer to start of key. + key will contain the key. + return_key_length will contain the length of key + after_key will contain the position to where the next key starts + */ + +uchar *_mi_find_half_pos(uint nod_flag, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key) +{ + uint keys,length,key_ref_length; + uchar *end,*lastpos; + DBUG_ENTER("_mi_find_half_pos"); + + key_ref_length=2+nod_flag; + length=mi_getint(page)-key_ref_length; + page+=key_ref_length; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + key_ref_length=keyinfo->keylength+nod_flag; + keys=length/(key_ref_length*2); + *return_key_length=keyinfo->keylength; + end=page+keys*key_ref_length; + *after_key=end+key_ref_length; + memcpy(key,end,key_ref_length); + DBUG_RETURN(end); + } + + end=page+length/2-key_ref_length; /* This is aprox. half */ + *key='\0'; + do + { + lastpos=page; + if (!(length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key))) + DBUG_RETURN(0); + } while (page < end); + *return_key_length=length; + *after_key=page; + DBUG_PRINT("exit",("returns: %lx page: %lx half: %lx",lastpos,page,end)); + DBUG_RETURN(lastpos); +} /* _mi_find_half_pos */ + + + /* + Split buffer at last key + Returns pointer to the start of the key before the last key + key will contain the last key + */ + +static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key) +{ + uint keys,length,last_length,key_ref_length; + uchar *end,*lastpos,*prevpos; + uchar key_buff[MI_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_find_last_pos"); + + key_ref_length=2; + length=mi_getint(page)-key_ref_length; + page+=key_ref_length; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + keys=length/keyinfo->keylength-2; + *return_key_length=length=keyinfo->keylength; + end=page+keys*length; + *after_key=end+length; + memcpy(key,end,length); + DBUG_RETURN(end); + } + + LINT_INIT(prevpos); + LINT_INIT(last_length); + end=page+length-key_ref_length; + *key='\0'; + length=0; + lastpos=page; + while (page < end) + { + prevpos=lastpos; lastpos=page; + last_length=length; + memcpy(key, key_buff, length); /* previous key */ + if (!(length=(*keyinfo->get_key)(keyinfo,0,&page,key_buff))) + { + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } + *return_key_length=last_length; + *after_key=lastpos; + DBUG_PRINT("exit",("returns: %lx page: %lx end: %lx",prevpos,page,end)); + DBUG_RETURN(prevpos); +} /* _mi_find_last_pos */ + + + /* Balance page with not packed keys with page on right/left */ + /* returns 0 if balance was done */ + +static int _mi_balance_page(register MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key, uchar *curr_buff, uchar *father_buff, + uchar *father_key_pos, my_off_t father_page) +{ + my_bool right; + uint k_length,father_length,father_keylength,nod_flag,curr_keylength, + right_length,left_length,new_right_length,new_left_length,extra_length, + length,keys; + uchar *pos,*buff,*extra_buff; + my_off_t next_page,new_pos; + byte tmp_part_key[MI_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_balance_page"); + + k_length=keyinfo->keylength; + father_length=mi_getint(father_buff); + father_keylength=k_length+info->s->base.key_reflength; + nod_flag=mi_test_if_nod(curr_buff); + curr_keylength=k_length+nod_flag; + info->page_changed=1; + + if ((father_key_pos != father_buff+father_length && (info->s->rnd++ & 1)) || + father_key_pos == father_buff+2+info->s->base.key_reflength) + { + right=1; + next_page= _mi_kpos(info->s->base.key_reflength, + father_key_pos+father_keylength); + buff=info->buff; + DBUG_PRINT("test",("use right page: %lu",next_page)); + } + else + { + right=0; + father_key_pos-=father_keylength; + next_page= _mi_kpos(info->s->base.key_reflength,father_key_pos); + /* Fix that curr_buff is to left */ + buff=curr_buff; curr_buff=info->buff; + DBUG_PRINT("test",("use left page: %lu",next_page)); + } /* father_key_pos ptr to parting key */ + + if (!_mi_fetch_keypage(info,keyinfo,next_page,info->buff,0)) + goto err; + DBUG_DUMP("next",(byte*) info->buff,mi_getint(info->buff)); + + /* Test if there is room to share keys */ + + left_length=mi_getint(curr_buff); + right_length=mi_getint(buff); + keys=(left_length+right_length-4-nod_flag*2)/curr_keylength; + + if ((right ? right_length : left_length) + curr_keylength <= + keyinfo->block_length) + { /* Merge buffs */ + new_left_length=2+nod_flag+(keys/2)*curr_keylength; + new_right_length=2+nod_flag+((keys+1)/2)*curr_keylength; + mi_putint(curr_buff,new_left_length,nod_flag); + mi_putint(buff,new_right_length,nod_flag); + + if (left_length < new_left_length) + { /* Move keys buff -> leaf */ + pos=curr_buff+left_length; + memcpy((byte*) pos,(byte*) father_key_pos, (size_t) k_length); + memcpy((byte*) pos+k_length, (byte*) buff+2, + (size_t) (length=new_left_length - left_length - k_length)); + pos=buff+2+length; + memcpy((byte*) father_key_pos,(byte*) pos,(size_t) k_length); + bmove((byte*) buff+2,(byte*) pos+k_length,new_right_length); + } + else + { /* Move keys -> buff */ + + bmove_upp((byte*) buff+new_right_length,(byte*) buff+right_length, + right_length-2); + length=new_right_length-right_length-k_length; + memcpy((byte*) buff+2+length,father_key_pos,(size_t) k_length); + pos=curr_buff+new_left_length; + memcpy((byte*) father_key_pos,(byte*) pos,(size_t) k_length); + memcpy((byte*) buff+2,(byte*) pos+k_length,(size_t) length); + } + + if (_mi_write_keypage(info,keyinfo,next_page,info->buff) || + _mi_write_keypage(info,keyinfo,father_page,father_buff)) + goto err; + DBUG_RETURN(0); + } + + /* curr_buff[] and buff[] are full, lets split and make new nod */ + + extra_buff=info->buff+info->s->base.max_key_block_length; + new_left_length=new_right_length=2+nod_flag+(keys+1)/3*curr_keylength; + if (keys == 5) /* Too few keys to balance */ + new_left_length-=curr_keylength; + extra_length=nod_flag+left_length+right_length- + new_left_length-new_right_length-curr_keylength; + DBUG_PRINT("info",("left_length: %d right_length: %d new_left_length: %d new_right_length: %d extra_length: %d", + left_length, right_length, + new_left_length, new_right_length, + extra_length)); + mi_putint(curr_buff,new_left_length,nod_flag); + mi_putint(buff,new_right_length,nod_flag); + mi_putint(extra_buff,extra_length+2,nod_flag); + + /* move first largest keys to new page */ + pos=buff+right_length-extra_length; + memcpy((byte*) extra_buff+2,pos,(size_t) extra_length); + /* Save new parting key */ + memcpy(tmp_part_key, pos-k_length,k_length); + /* Make place for new keys */ + bmove_upp((byte*) buff+new_right_length,(byte*) pos-k_length, + right_length-extra_length-k_length-2); + /* Copy keys from left page */ + pos= curr_buff+new_left_length; + memcpy((byte*) buff+2,(byte*) pos+k_length, + (size_t) (length=left_length-new_left_length-k_length)); + /* Copy old parting key */ + memcpy((byte*) buff+2+length,father_key_pos,(size_t) k_length); + + /* Move new parting keys up to caller */ + memcpy((byte*) (right ? key : father_key_pos),pos,(size_t) k_length); + memcpy((byte*) (right ? father_key_pos : key),tmp_part_key, k_length); + + if ((new_pos=_mi_new(info,keyinfo)) == HA_OFFSET_ERROR) + goto err; + _mi_kpointer(info,key+k_length,new_pos); + if (_mi_write_keypage(info,keyinfo,(right ? new_pos : next_page), + info->buff) || + _mi_write_keypage(info,keyinfo,(right ? next_page : new_pos),extra_buff)) + goto err; + + DBUG_RETURN(1); /* Middle key up */ + +err: + DBUG_RETURN(-1); +} /* _mi_balance_page */ diff --git a/myisam/myisamchk.c b/myisam/myisamchk.c new file mode 100644 index 00000000000..4f2288322e8 --- /dev/null +++ b/myisam/myisamchk.c @@ -0,0 +1,1365 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Descript, check and repair of ISAM tables */ + +#include "myisamdef.h" + +#include <m_ctype.h> +#include <stdarg.h> +#include <getopt.h> +#include <assert.h> +#ifdef HAVE_SYS_VADVICE_H +#include <sys/vadvise.h> +#endif +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +SET_STACK_SIZE(9000) /* Minimum stack size for program */ + +#ifndef USE_RAID +#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G) +#define my_raid_delete(A,B,C) my_delete(A,B) +#endif + +static uint decode_bits; +static char **default_argv; +static const char *load_default_groups[]= { "myisamchk", 0 }; +static const char *set_charset_name; +static uint8 set_charset_number; +static CHARSET_INFO *set_charset; + +static const char *type_names[]= +{ "?","char","binary", "short", "long", "float", + "double","number","unsigned short", + "unsigned long","longlong","ulonglong","int24", + "uint24","int8","varchar", "varbin","?", + "?"}; + +static const char *prefix_packed_txt="packed ", + *bin_packed_txt="prefix ", + *diff_txt="stripped ", + *null_txt="NULL", + *blob_txt="BLOB "; + +static const char *field_pack[]= +{"","no endspace", "no prespace", + "no zeros", "blob", "constant", "table-lockup", + "always zero","varchar","unique-hash","?","?"}; + + +static void get_options(int *argc,char * * *argv); +static void print_version(void); +static void usage(void); +static int myisamchk(MI_CHECK *param, char *filename); +static void descript(MI_CHECK *param, register MI_INFO *info, my_string name); +static int mi_sort_records(MI_CHECK *param, + register MI_INFO *info, my_string name, + uint sort_key, + my_bool write_info, + my_bool update_index); +static int sort_record_index(MI_CHECK *param,MI_INFO *info,MI_KEYDEF *keyinfo, + my_off_t page,uchar *buff,uint sortkey, + File new_file, my_bool update_index); + +MI_CHECK check_param; + + /* Main program */ + +int main(int argc, char **argv) +{ + int error; + MY_INIT(argv[0]); + + myisamchk_init(&check_param); + check_param.opt_lock_memory=1; /* Lock memory if possible */ + check_param.using_global_keycache = 0; + get_options(&argc,(char***) &argv); + myisam_quick_table_bits=decode_bits; + error=0; + while (--argc >= 0) + { + error|= myisamchk(&check_param, *(argv++)); + VOID(fflush(stdout)); + VOID(fflush(stderr)); + if ((check_param.error_printed | check_param.warning_printed) && + (check_param.testflag & T_FORCE_CREATE) && + (!(check_param.testflag & (T_REP | T_REP_BY_SORT | T_SORT_RECORDS | + T_SORT_INDEX)))) + { + uint old_testflag=check_param.testflag; + check_param.testflag|=T_REP; + check_param.testflag&= ~T_EXTEND; /* Don't needed */ + error|=myisamchk(&check_param, argv[-1]); + check_param.testflag= old_testflag; + VOID(fflush(stdout)); + VOID(fflush(stderr)); + } + if (argc && (!(check_param.testflag & T_SILENT) || check_param.testflag & T_INFO)) + { + puts("\n---------\n"); + VOID(fflush(stdout)); + } + } + if (check_param.total_files > 1) + { /* Only if descript */ + char buff[22],buff2[22]; + if (!(check_param.testflag & T_SILENT) || check_param.testflag & T_INFO) + puts("\n---------\n"); + printf("\nTotal of all %d ISAM-files:\nData records: %9s Deleted blocks: %9s\n",check_param.total_files,llstr(check_param.total_records,buff), + llstr(check_param.total_deleted,buff2)); + } + free_defaults(default_argv); + my_end(check_param.testflag & T_INFO ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + exit(error); +#ifndef _lint + return 0; /* No compiler warning */ +#endif +} /* main */ + + +static CHANGEABLE_VAR changeable_vars[] = { + { "key_buffer_size",(long*) &check_param.use_buffers,(long) USE_BUFFER_INIT, + (long) MALLOC_OVERHEAD, (long) ~0L,(long) MALLOC_OVERHEAD,(long) IO_SIZE }, + { "read_buffer_size", (long*) &check_param.read_buffer_length,(long) READ_BUFFER_INIT, + (long) MALLOC_OVERHEAD,(long) ~0L,(long) MALLOC_OVERHEAD,(long) 1L }, + { "write_buffer_size", (long*) &check_param.write_buffer_length,(long) READ_BUFFER_INIT, + (long) MALLOC_OVERHEAD,(long) ~0L,(long) MALLOC_OVERHEAD,(long) 1L }, + { "sort_buffer_size",(long*) &check_param.sort_buffer_length,(long) SORT_BUFFER_INIT, + (long) (MIN_SORT_BUFFER+MALLOC_OVERHEAD),(long) ~0L, + (long) MALLOC_OVERHEAD,(long) 1L }, + { "sort_key_blocks",(long*) &check_param.sort_key_blocks,BUFFERS_WHEN_SORTING,4L,100L,0L, + 1L }, + { "decode_bits",(long*) &decode_bits,9L,4L,17L,0L,1L }, + { NullS,(long*) 0,0L,0L,0L,0L,0L,} }; + +enum options {OPT_CHARSETS_DIR=256, OPT_SET_CHARSET}; + + +static struct option long_options[] = +{ + {"analyze", no_argument, 0, 'a'}, + {"block-search", required_argument, 0, 'b'}, + {"character-sets-dir",required_argument,0, OPT_CHARSETS_DIR}, + {"check", no_argument, 0, 'c'}, + {"check-only-changed",no_argument, 0, 'C'}, +#ifndef DBUG_OFF + {"debug", required_argument, 0, '#'}, +#endif + {"description", no_argument, 0, 'd'}, + {"data-file-length", required_argument, 0, 'D'}, + {"extend-check", no_argument, 0, 'e'}, + {"fast", no_argument, 0, 'F'}, + {"force", no_argument, 0, 'f'}, + {"help", no_argument, 0, '?'}, + {"information", no_argument, 0, 'i'}, + {"keys-used", required_argument, 0, 'k'}, + {"medium-check", no_argument, 0, 'm'}, + {"no-symlinks", no_argument, 0, 'l'}, + {"quick", no_argument, 0, 'q'}, + {"read-only", no_argument, 0, 'T'}, + {"recover", no_argument, 0, 'r'}, + {"safe-recover", no_argument, 0, 'o'}, + {"set-auto-increment",optional_argument, 0, 'A'}, + {"set-character-set",required_argument,0,OPT_SET_CHARSET}, + {"set-variable", required_argument, 0, 'O'}, + {"silent", no_argument, 0, 's'}, + {"sort-index", no_argument, 0, 'S'}, + {"sort-records", required_argument, 0, 'R'}, + {"tmpdir", required_argument, 0, 't'}, + {"update-state", no_argument, 0, 'U'}, + {"unpack", no_argument, 0, 'u'}, + {"verbose", no_argument, 0, 'v'}, + {"version", no_argument, 0, 'V'}, + {"wait", no_argument, 0, 'w'}, + {0, 0, 0, 0} +}; + +static void print_version(void) +{ + printf("%s Ver 1.28 for %s at %s\n",my_progname,SYSTEM_TYPE, + MACHINE_TYPE); +} + +static void usage(void) +{ + uint i; + print_version(); + puts("By Monty, for your professional use"); + puts("This software comes with NO WARRANTY: see the PUBLIC for details.\n"); + puts("Description, check and repair of ISAM tables."); + puts("Used without options all tables on the command will be checked for errors"); + printf("Usage: %s [OPTIONS] tables[.MYI]\n", my_progname); + puts("\nGlobal options:\n\ + -#, --debug=... Output debug log. Often this is 'd:t:o,filename`\n\ + -?, --help Display this help and exit.\n\ + -O, --set-variable var=option\n\ + Change the value of a variable.\n\ + -s, --silent Only print errors. One can use two -s to make\n\ + myisamchk very silent\n\ + -v, --verbose Print more information. This can be used with\n\ + --describe and --check. Use many -v for more verbosity!\n\ + -V, --version Print version and exit.\n\ + -w, --wait Wait if table is locked.\n"); + + puts("Check options (check is the default action for myisamchk):\n\ + -c, --check Check table for errors\n\ + -e, --extend-check Check the table VERY throughly. Only use this in\n\ + extreme cases as myisamchk should normally be able to\n\ + find out if the table is ok even without this switch\n\ + -F, --fast Check only tables that hasn't been closed properly\n\ + -C, --check-changed-tables\n\ + Check only tables that has changed since last check\n\ + -f, --force Restart with -r if there are any errors in the table\n\ + -i, --information Print statistics information about table that is checked\n\ + -m, --medium-check Faster than extended-check, but only finds 99.99% of\n\ + all errors. Should however be good enough for most cases\n\ + -U --update-state Mark tables as crashed if you find any errors\n\ + -T, --read-only Don't mark table as checked\n"); + + puts("Repair options (When using -r or -o) \n\ + -D, --data-file-length=# Max length of data file (when recreating data\n\ + file when it's full)\n\ + -e, --extend-check Try to recover every possible row from the data file\n\ + Normally this will also find a lot of garbage rows;\n\ + Don't use this option if you are not totally desperate.\n\ + -f, --force Overwrite old temporary files.\n\ + -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ + bit mask of which keys to use. This can be used to\n\ + get faster inserts!\n\ + -l, --no-symlinks Do not follow symbolic links. Normally\n\ + myisamchk repairs the table a symlink points at.\n\ + -r, --recover Can fix almost anything except unique keys that aren't\n\ + unique.\n\ + -o, --safe-recover Uses old recovery method; Slower than '-r' but can\n\ + handle a couple of cases where '-r' reports that it can't\n\ + fix the data file.\n\ + --character-sets-dir=...\n\ + Directory where character sets are\n\ + --set-character-set=name\n\ + Change the character set used by the index\n\ + -t, --tmpdir=path Path for temporary files\n\ + -q, --quick Faster repair by not modifying the data file.\n\ + One can give a second '-q' to force myisamchk to\n\ + modify the original datafile in case of duplicate keys\n\ + -u, --unpack Unpack file packed with myisampack.\n\ +"); + + puts("Other actions:\n\ + -a, --analyze Analyze distribution of keys. Will make some joins in\n\ + MySQL faster. You can check the calculated distribution\n\ + by using '--describe --verbose table_name'.\n\ + -d, --description Prints some information about table.\n\ + -A, --set-auto-increment[=value]\n\ + Force auto_increment to start at this or higher value\n\ + If no value is given, then sets the next auto_increment\n\ + value to the highest used value for the auto key + 1.\n\ + -S, --sort-index Sort index blocks. This speeds up 'read-next' in\n\ + applications\n\ + -R, --sort-records=#\n\ + Sort records according to an index. This makes your\n\ + data much more localized and may speed up things\n\ + (It may be VERY slow to do a sort the first time!)"); + + print_defaults("my",load_default_groups); + printf("\nPossible variables for option --set-variable (-O) are:\n"); + for (i=0; changeable_vars[i].name ; i++) + printf("%-20s current value: %lu\n", + changeable_vars[i].name, + *changeable_vars[i].varptr); +} + + + /* Read options */ + +static void get_options(register int *argc,register char ***argv) +{ + int c,option_index=0; + uint old_testflag; + + load_defaults("my",load_default_groups,argc,argv); + default_argv= *argv; + set_all_changeable_vars(changeable_vars); + if (isatty(fileno(stdout))) + check_param.testflag|=T_WRITE_LOOP; + while ((c=getopt_long(*argc,*argv,"acCdeif?lqrmosSTuUvVw#:b:D:k:O:R:A::t:", + long_options, &option_index)) != EOF) + { + switch(c) { + case 'a': + check_param.testflag|= T_STATISTICS; + break; + case 'A': + if (optarg) + check_param.auto_increment_value=strtoull(optarg,NULL,0); + else + check_param.auto_increment_value=0; /* Set to max used value */ + check_param.testflag|= T_AUTO_INC; + break; + case 'b': + check_param.search_after_block=strtoul(optarg,NULL,10); + break; + case 'c': + check_param.testflag|= T_CHECK; + break; + case 'C': + check_param.testflag|= T_CHECK | T_CHECK_ONLY_CHANGED; + break; + case 'D': + check_param.max_data_file_length=strtoll(optarg,NULL,10); + break; + case 's': /* silent */ + if (check_param.testflag & T_SILENT) + check_param.testflag|=T_VERY_SILENT; + check_param.testflag|= T_SILENT; + check_param.testflag&= ~T_WRITE_LOOP; + break; + case 'w': + check_param.testflag|= T_WAIT_FOREVER; + break; + case 'd': /* description if isam-file */ + check_param.testflag|= T_DESCRIPT; + break; + case 'e': /* extend check */ + check_param.testflag|= T_EXTEND; + break; + case 'i': + check_param.testflag|= T_INFO; + break; + case 'f': + check_param.tmpfile_createflag= O_RDWR | O_TRUNC; + check_param.testflag|=T_FORCE_CREATE; + break; + case 'F': + check_param.testflag|=T_FAST; + break; + case 'k': + check_param.keys_in_use= (ulonglong) strtoll(optarg,NULL,10); + break; + case 'l': + check_param.opt_follow_links=0; + break; + case 'm': + check_param.testflag|= T_MEDIUM; /* Medium check */ + break; + case 'r': /* Repair table */ + check_param.testflag= (check_param.testflag & ~T_REP) | T_REP_BY_SORT; + break; + case 'o': + check_param.testflag= (check_param.testflag & ~T_REP_BY_SORT) | T_REP; + my_disable_async_io=1; /* More safety */ + break; + case 'q': + check_param.opt_rep_quick++; + break; + case 'u': + check_param.testflag|= T_UNPACK | T_REP_BY_SORT; + break; + case 'v': /* Verbose */ + check_param.testflag|= T_VERBOSE; + check_param.verbose++; + break; + case 'O': + if (set_changeable_var(optarg, changeable_vars)) + { + usage(); + exit(1); + } + break; + case 'R': /* Sort records */ + old_testflag=check_param.testflag; + check_param.testflag|= T_SORT_RECORDS; + check_param.opt_sort_key=(uint) atoi(optarg)-1; + if (check_param.opt_sort_key >= MI_MAX_KEY) + { + fprintf(stderr, + "The value of the sort key is bigger than max key: %d.\n", + MI_MAX_KEY); + exit(1); + } + break; + case 'S': /* Sort index */ + old_testflag=check_param.testflag; + check_param.testflag|= T_SORT_INDEX; + break; + case 't': + check_param.tmpdir=optarg; + break; + case 'T': + check_param.testflag|= T_READONLY; + break; + case 'U': + check_param.testflag|= T_UPDATE_STATE; + break; + case '#': + DBUG_PUSH(optarg ? optarg : "d:t:o,/tmp/isamchk"); + break; + case 'V': + print_version(); + exit(0); + case OPT_CHARSETS_DIR: + charsets_dir = optarg; + break; + case OPT_SET_CHARSET: + set_charset_name=optarg; + break; + case '?': + usage(); + exit(0); + } + } + (*argc)-=optind; + (*argv)+=optind; + if (*argc == 0) + { + usage(); + exit(-1); + } + if ((check_param.testflag & T_UNPACK) && + (check_param.opt_rep_quick || (check_param.testflag & T_SORT_RECORDS))) + { + VOID(fprintf(stderr, + "%s: --unpack can't be used with --quick or --sort-records\n", + my_progname)); + exit(1); + } + if ((check_param.testflag & T_READONLY) && + (check_param.testflag & + (T_REP_BY_SORT | T_REP | T_STATISTICS | T_AUTO_INC | + T_SORT_RECORDS | T_SORT_INDEX))) + { + VOID(fprintf(stderr, + "%s: Can't use --readonly when repairing or sorting\n", + my_progname)); + exit(1); + } + + if (set_charset_name) + if (!(set_charset=get_charset_by_name(set_charset_name, MYF(MY_WME)))) + exit(1); + + return; +} /* get options */ + + + /* Check table */ + +static int myisamchk(MI_CHECK *param, my_string filename) +{ + int error,lock_type,recreate; + int rep_quick= param->opt_rep_quick; + uint raid_chunks; + MI_INFO *info; + File datafile; + char fixed_name[FN_REFLEN]; + char llbuff[22],llbuff2[22]; + MYISAM_SHARE *share; + DBUG_ENTER("myisamchk"); + + param->out_flag=error=param->warning_printed=param->error_printed= + recreate=0; + datafile=0; + param->isam_file_name=filename; /* For error messages */ + if (!(info=mi_open(filename, + (param->testflag & (T_DESCRIPT | T_READONLY)) ? + O_RDONLY : O_RDWR, + (param->testflag & T_WAIT_FOREVER) ? + HA_OPEN_WAIT_IF_LOCKED : + (param->testflag & T_DESCRIPT) ? + HA_OPEN_IGNORE_IF_LOCKED : HA_OPEN_ABORT_IF_LOCKED))) + { + /* Avoid twice printing of isam file name */ + param->error_printed=1; + switch (my_errno) { + case HA_ERR_CRASHED: + mi_check_print_error(param,"'%s' is not a MyISAM-table",filename); + break; + case HA_ERR_OLD_FILE: + mi_check_print_error(param,"'%s' is a old type of MyISAM-table", filename); + break; + case HA_ERR_END_OF_FILE: + mi_check_print_error(param,"Couldn't read complete header from '%s'", filename); + break; + case EAGAIN: + mi_check_print_error(param,"'%s' is locked. Use -w to wait until unlocked",filename); + break; + case ENOENT: + mi_check_print_error(param,"File '%s' doesn't exist",filename); + break; + case EACCES: + mi_check_print_error(param,"You don't have permission to use '%s'",filename); + break; + default: + mi_check_print_error(param,"%d when opening MyISAM-table '%s'", + my_errno,filename); + break; + } + DBUG_RETURN(1); + } + share=info->s; + share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */ + share->r_locks=0; + raid_chunks=share->base.raid_chunks; + + + /* + Skipp the checking of the file if: + We are using --fast and the table is closed properly + We are using --check-only-changed-tables and the table hasn't changed + */ + if ((param->testflag & T_CHECK_ONLY_CHANGED) && !share->state.changed || + (param->testflag & T_FAST) && share->state.open_count == 0) + { + if (!(param->testflag & T_SILENT) || param->testflag & T_INFO) + printf("MyISAM file: %s is already checked\n",filename); + if (mi_close(info)) + { + mi_check_print_error(param,"%d when closing MyISAM-table '%s'", + my_errno,filename); + DBUG_RETURN(1); + } + DBUG_RETURN(0); + } + + if ((param->testflag & (T_REP_BY_SORT | T_REP | T_STATISTICS | + T_SORT_RECORDS | T_SORT_INDEX)) && + (((param->testflag & T_UNPACK) && + share->data_file_type == COMPRESSED_RECORD) || + mi_uint2korr(share->state.header.state_info_length) != + MI_STATE_INFO_SIZE || + mi_uint2korr(share->state.header.base_info_length) != + MI_BASE_INFO_SIZE || + ((param->keys_in_use & ~share->state.key_map) & + (((ulonglong) 1L << share->base.keys)-1)) || + test_if_almost_full(info) || + info->s->state.header.file_version[3] != myisam_file_magic[3] || + (set_charset && set_charset_number != share->state.header.language))) + { + check_param.language=set_charset_number; + if (recreate_table(&check_param, &info,filename)) + { + VOID(fprintf(stderr, + "MyISAM-table '%s' is not fixed because of errors\n", + filename)); + return(-1); + } + recreate=1; + if (!(param->testflag & (T_REP | T_REP_BY_SORT))) + { + param->testflag|=T_REP_BY_SORT; /* if only STATISTICS */ + if (!(param->testflag & T_SILENT)) + printf("- '%s' has old table-format. Recreating index\n",filename); + if (!rep_quick) + rep_quick=1; + } + share=info->s; + share->r_locks=0; + } + + if (param->testflag & T_DESCRIPT) + { + param->total_files++; + param->total_records+=info->state->records; + param->total_deleted+=info->state->del; + descript(&check_param, info, filename); + } + else + { + if (!(param->testflag & T_READONLY)) + lock_type = F_WRLCK; /* table is changed */ + else + lock_type= F_RDLCK; + if (info->lock_type == F_RDLCK) + info->lock_type=F_UNLCK; /* Read only table */ + if (_mi_readinfo(info,lock_type,0)) + { + mi_check_print_error(param,"Can't lock indexfile of '%s', error: %d", + filename,my_errno); + param->error_printed=0; + goto end2; + } + share->w_locks++; /* Mark for writeinfo */ + info->lock_type= F_EXTRA_LCK; /* Simulate as locked */ + info->tmp_lock_type=lock_type; + datafile=info->dfile; + + if (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX)) + { + if (param->testflag & (T_REP+T_REP_BY_SORT)) + share->state.key_map= (((ulonglong) 1 << share->base.keys)-1) + & param->keys_in_use; + VOID(fn_format(fixed_name,filename,"",MI_NAME_IEXT, + 4+ (param->opt_follow_links ? 16 : 0))); + + if (rep_quick && chk_del(&check_param, info, + param->testflag & ~T_VERBOSE)) + { + if (param->testflag & T_FORCE_CREATE) + { + rep_quick=0; + mi_check_print_info(param,"Creating new data file\n"); + } + else + { + error=1; + mi_check_print_error(param, + "Quick-recover aborted; Run recovery without switch 'q'"); + } + } + if (!error) + { + if (param->testflag & T_REP_BY_SORT && + (share->state.key_map || + (rep_quick && !param->keys_in_use && !recreate))) + error=mi_repair_by_sort(&check_param,info,fixed_name,rep_quick); + else if (param->testflag & (T_REP | T_REP_BY_SORT)) + error=mi_repair(&check_param, info,fixed_name,rep_quick); + } + if (!error && param->testflag & T_SORT_RECORDS) + { + if (param->out_flag & O_NEW_DATA) + { /* Change temp file to org file */ + VOID(my_close(info->dfile,MYF(MY_WME))); /* Close new file */ + error|=change_to_newfile(fixed_name,MI_NAME_DEXT,DATA_TMP_EXT, + raid_chunks); +#ifdef USE_RAID + if (share->base.raid_type) + { + mi_check_print_info(&check_param,"Opening as RAID-ed table\n"); + info->dfile=my_raid_open(fn_format(param->temp_filename, + fixed_name,"", + MI_NAME_DEXT, 2+4), + O_RDWR | O_SHARE, + share->base.raid_type, + raid_chunks, + share->base.raid_chunksize, + MYF(MY_WME | MY_RAID)); + } + else +#endif + info->dfile=my_open(fn_format(param->temp_filename, + fixed_name,"", + MI_NAME_DEXT,2+4), + O_RDWR | O_SHARE, + MYF(MY_WME)); + if (info->dfile < 0) + error=1; + param->out_flag&= ~O_NEW_DATA; /* We are using new datafile */ + param->read_cache.file=info->dfile; + } + if (! error) + { + uint key; + /* + We can't update the index in mi_sort_records if we have a + prefix compressed index + */ + my_bool update_index=1; + for (key=0 ; key < share->base.keys; key++) + if (share->keyinfo[key].flag & HA_BINARY_PACK_KEY) + update_index=0; + + error=mi_sort_records(param,info,fixed_name,param->opt_sort_key, + (my_bool) !(param->testflag & T_REP), + update_index); + datafile=info->dfile; /* This is now locked */ + if (!error && !update_index) + { + if (check_param.verbose) + puts("Table had a compressed index; We must now recreate the index"); + error=mi_repair_by_sort(&check_param,info,fixed_name,1); + } + } + } + if (!error && param->testflag & T_SORT_INDEX) + error=mi_sort_index(param,info,fixed_name); + if (!error) + share->state.changed=0; + else + mi_mark_crashed(info); + } + else if ((param->testflag & T_CHECK) || !(param->testflag & T_AUTO_INC)) + { + if (!(param->testflag & T_SILENT) || param->testflag & T_INFO) + printf("Checking MyISAM file: %s\n",filename); + if (!(param->testflag & T_SILENT)) + printf("Data records: %7s Deleted blocks: %7s\n", + llstr(info->state->records,llbuff), + llstr(info->state->del,llbuff2)); + if (share->state.open_count) + { + mi_check_print_warning(param, + "%d clients is using or hasn't closed the table properly", + share->state.open_count); + } + share->state.key_map &=param->keys_in_use; + error =chk_size(param,info); + if (!error || !(param->testflag & (T_FAST | T_FORCE_CREATE))) + error|=chk_del(param, info,param->testflag); + if (!error || !(param->testflag & (T_FAST | T_FORCE_CREATE))) + { + error|=chk_key(param, info); + if (!error && (param->testflag & (T_STATISTICS | T_AUTO_INC))) + error=update_state_info(param, info, UPDATE_STAT); + } + if ((!rep_quick && !error) || + !(param->testflag & (T_FAST | T_FORCE_CREATE))) + { + if (param->testflag & (T_EXTEND | T_MEDIUM)) + VOID(init_key_cache(param->use_buffers,(uint) NEAD_MEM)); + VOID(init_io_cache(¶m->read_cache,datafile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1, + MYF(MY_WME))); + lock_memory(param); + if ((info->s->options & (HA_OPTION_PACK_RECORD | + HA_OPTION_COMPRESS_RECORD)) || + (param->testflag & (T_EXTEND | T_MEDIUM))) + error|=chk_data_link(param, info, param->testflag & T_EXTEND); + error|=flush_blocks(param,share->kfile); + VOID(end_io_cache(¶m->read_cache)); + } + if (!error) + { + if (share->state.changed && (param->testflag & T_UPDATE_STATE)) + info->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + share->state.changed=0; + } + else if (!mi_is_crashed(info) && + (param->testflag & T_UPDATE_STATE)) + { /* Mark crashed */ + mi_mark_crashed(info); + info->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + } + } + } + if ((param->testflag & T_AUTO_INC) || + ((param->testflag & (T_REP | T_REP_BY_SORT)) && + info->s->base.auto_key)) + update_auto_increment_key(param, info, + (my_bool) !test(param->testflag & T_AUTO_INC)); + + if (!(param->testflag & T_DESCRIPT)) + { + if (info->update & HA_STATE_CHANGED && ! (param->testflag & T_READONLY)) + error|=update_state_info(param, info, + UPDATE_OPEN_COUNT | + (((param->testflag & (T_REP | T_REP_BY_SORT)) ? + UPDATE_TIME | UPDATE_STAT : 0) | + ((param->testflag & T_SORT_RECORDS) ? + UPDATE_SORT : 0))); + VOID(lock_file(param, share->kfile,0L,F_UNLCK,"indexfile",filename)); + info->update&= ~HA_STATE_CHANGED; + } + share->w_locks--; +end2: + if (mi_close(info)) + { + mi_check_print_error(param,"%d when closing MyISAM-table '%s'",my_errno,filename); + DBUG_RETURN(1); + } + if (error == 0) + { + if (param->out_flag & O_NEW_DATA) + error|=change_to_newfile(fixed_name,MI_NAME_DEXT,DATA_TMP_EXT, + raid_chunks); + if (param->out_flag & O_NEW_INDEX) + error|=change_to_newfile(fixed_name,MI_NAME_IEXT,INDEX_TMP_EXT,0); + } + VOID(fflush(stdout)); VOID(fflush(stderr)); + if (param->error_printed) + { + if (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX)) + { + VOID(fprintf(stderr, + "MyISAM-table '%s' is not fixed because of errors\n", + filename)); + if (check_param.testflag & (T_REP_BY_SORT | T_REP)) + VOID(fprintf(stderr, + "Try fixing it by using the --safe-recover (-o) option\n")); + } + else if (!(param->error_printed & 2) && + !(param->testflag & T_FORCE_CREATE)) + VOID(fprintf(stderr, + "MyISAM-table '%s' is corrupted\nFix it using switch \"-r\" or \"-o\"\n", + filename)); + } + else if (param->warning_printed && + ! (param->testflag & (T_REP+T_REP_BY_SORT+T_SORT_RECORDS+T_SORT_INDEX+ + T_FORCE_CREATE))) + VOID(fprintf(stderr, "MyISAM-table '%s' is usable but should be fixed\n", + filename)); + VOID(fflush(stderr)); + DBUG_RETURN(error); +} /* myisamchk */ + + + /* Write info about table */ + +static void descript(MI_CHECK *param, register MI_INFO *info, my_string name) +{ + uint key,keyseg_nr,field,start; + reg3 MI_KEYDEF *keyinfo; + reg2 MI_KEYSEG *keyseg; + reg4 const char *text; + char buff[40],length[10],*pos,*end; + enum en_fieldtype type; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + DBUG_ENTER("describe"); + + printf("\nMyISAM file: %s\n",name); + fputs("Record format: ",stdout); + if (share->options & HA_OPTION_COMPRESS_RECORD) + puts("Compressed"); + else if (share->options & HA_OPTION_PACK_RECORD) + puts("Packed"); + else + puts("Fixed length"); + printf("Character set: %s (%d)\n", + get_charset_name(share->state.header.language), + share->state.header.language); + + if (param->testflag & T_VERBOSE) + { + printf("File-version: %d\n", + (int) share->state.header.file_version[3]); + if (share->state.create_time) + { + get_date(buff,1,share->state.create_time); + printf("Creation time: %s\n",buff); + } + if (share->state.check_time) + { + get_date(buff,1,share->state.check_time); + printf("Recover time: %s\n",buff); + } + printf("Status: %s\n", + share->state.changed & 2 ? "crashed" : + share->state.open_count ? "open" : + share->state.changed ? "changed" : "checked"); + if (share->base.auto_key) + { + printf("Auto increment key: %13d Last value: %13s\n", + share->base.auto_key, + llstr(share->state.auto_increment,llbuff)); + } + if (share->base.raid_type) + { + printf("RAID: Type: %u Chunks: %u Chunksize: %lu\n", + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize); + } + if (share->options & HA_OPTION_CHECKSUM) + printf("Using checksums\n"); + if (share->options & HA_OPTION_DELAY_KEY_WRITE) + printf("Keys are only flushed at close\n"); + + } + printf("Data records: %13s Deleted blocks: %13s\n", + llstr(info->state->records,llbuff),llstr(info->state->del,llbuff2)); + if (param->testflag & T_SILENT) + DBUG_VOID_RETURN; /* This is enough */ + + if (param->testflag & T_VERBOSE) + { +#ifdef USE_RELOC + printf("Init-relocation: %13s\n",llstr(share->base.reloc,llbuff)); +#endif + printf("Datafile parts: %13s Deleted data: %13s\n", + llstr(share->state.split,llbuff), + llstr(info->state->empty,llbuff2)); + printf("Datafile pointer (bytes):%9d Keyfile pointer (bytes):%9d\n", + share->rec_reflength,share->base.key_reflength); + printf("Datafile length: %13s Keyfile length: %13s\n", + llstr(info->state->data_file_length,llbuff), + llstr(info->state->key_file_length,llbuff2)); + + if (info->s->base.reloc == 1L && info->s->base.records == 1L) + puts("This is a one-record table"); + else + { + if (share->base.max_data_file_length != HA_OFFSET_ERROR || + share->base.max_key_file_length != HA_OFFSET_ERROR) + printf("Max datafile length: %13s Max keyfile length: %13s\n", + llstr(share->base.max_data_file_length-1,llbuff), + llstr(share->base.max_key_file_length-1,llbuff2)); + } + } + + printf("Recordlength: %13d\n",(int) share->base.pack_reclength); + if (share->state.key_map != (((ulonglong) 1 << share->base.keys) -1)) + { + longlong2str(share->state.key_map,buff,2); + printf("Using only keys '%s' of %d possibly keys\n", + buff, share->base.keys); + } + puts("\ntable description:"); + printf("Key Start Len Index Type"); + if (param->testflag & T_VERBOSE) + printf(" Rec/key Root Blocksize"); + VOID(putchar('\n')); + + for (key=keyseg_nr=0, keyinfo= &share->keyinfo[0] ; + key < share->base.keys; + key++,keyinfo++) + { + keyseg=keyinfo->seg; + if (keyinfo->flag & HA_NOSAME) text="unique "; + else text="multip."; + + pos=buff; + if (keyseg->flag & HA_REVERSE_SORT) + *pos++ = '-'; + pos=strmov(pos,type_names[keyseg->type]); + *pos++ = ' '; + *pos=0; + if (keyinfo->flag & HA_PACK_KEY) + pos=strmov(pos,prefix_packed_txt); + if (keyinfo->flag & HA_BINARY_PACK_KEY) + pos=strmov(pos,bin_packed_txt); + if (keyseg->flag & HA_SPACE_PACK) + pos=strmov(pos,diff_txt); + if (keyseg->flag & HA_BLOB_PART) + pos=strmov(pos,blob_txt); + if (keyseg->flag & HA_NULL_PART) + pos=strmov(pos,null_txt); + *pos=0; + + printf("%-4d%-6ld%-3d %-8s%-21s", + key+1,(long) keyseg->start+1,keyseg->length,text,buff); + if (share->state.key_root[key] != HA_OFFSET_ERROR) + llstr(share->state.key_root[key],buff); + else + buff[0]=0; + if (param->testflag & T_VERBOSE) + printf("%11lu %12s %10d", + share->state.rec_per_key_part[keyseg_nr++], + buff,keyinfo->block_length); + VOID(putchar('\n')); + while ((++keyseg)->type != HA_KEYTYPE_END) + { + pos=buff; + if (keyseg->flag & HA_REVERSE_SORT) + *pos++ = '-'; + pos=strmov(pos,type_names[keyseg->type]); + *pos++= ' '; + if (keyseg->flag & HA_SPACE_PACK) + pos=strmov(pos,diff_txt); + if (keyseg->flag & HA_BLOB_PART) + pos=strmov(pos,blob_txt); + if (keyseg->flag & HA_NULL_PART) + pos=strmov(pos,null_txt); + *pos=0; + printf(" %-6ld%-3d %-21s", + (long) keyseg->start+1,keyseg->length,buff); + if (param->testflag & T_VERBOSE) + printf("%11lu", share->state.rec_per_key_part[keyseg_nr++]); + VOID(putchar('\n')); + } + keyseg++; + } + if (share->state.header.uniques) + { + MI_UNIQUEDEF *uniqueinfo; + puts("\nUnique Key Start Len Nullpos Nullbit Type"); + for (key=0,uniqueinfo= &share->uniqueinfo[0] ; + key < share->state.header.uniques; key++, uniqueinfo++) + { + my_bool new_row=0; + char null_bit[8],null_pos[8]; + printf("%-8d%-5d",key+1,uniqueinfo->key+1); + for (keyseg=uniqueinfo->seg ; keyseg->type != HA_KEYTYPE_END ; keyseg++) + { + if (new_row) + fputs(" ",stdout); + null_bit[0]=null_pos[0]=0; + if (keyseg->null_bit) + { + sprintf(null_bit,"%d",keyseg->null_bit); + sprintf(null_pos,"%ld",(long) keyseg->null_pos+1); + } + printf("%-7ld%-5d%-9s%-10s%-30s\n", + (long) keyseg->start+1,keyseg->length, + null_pos,null_bit, + type_names[keyseg->type]); + new_row=1; + } + } + } + if (param->verbose > 1) + { + char null_bit[8],null_pos[8]; + printf("\nField Start Length Nullpos Nullbit Type"); + if (share->options & HA_OPTION_COMPRESS_RECORD) + printf(" Huff tree Bits"); + VOID(putchar('\n')); + start=1; + for (field=0 ; field < share->base.fields ; field++) + { + if (share->options & HA_OPTION_COMPRESS_RECORD) + type=share->rec[field].base_type; + else + type=(enum en_fieldtype) share->rec[field].type; + end=strmov(buff,field_pack[type]); + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + if (share->rec[field].pack_type & PACK_TYPE_SELECTED) + end=strmov(end,", not_always"); + if (share->rec[field].pack_type & PACK_TYPE_SPACE_FIELDS) + end=strmov(end,", no empty"); + if (share->rec[field].pack_type & PACK_TYPE_ZERO_FILL) + { + sprintf(end,", zerofill(%d)",share->rec[field].space_length_bits); + end=strend(end); + } + } + if (buff[0] == ',') + strmov(buff,buff+2); + int2str((long) share->rec[field].length,length,10); + null_bit[0]=null_pos[0]=0; + if (share->rec[field].null_bit) + { + sprintf(null_bit,"%d",share->rec[field].null_bit); + sprintf(null_pos,"%d",share->rec[field].null_pos+1); + } + printf("%-6d%-6d%-7s%-8s%-8s%-35s",field+1,start,length, + null_pos, null_bit, buff); + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + if (share->rec[field].huff_tree) + printf("%3d %2d", + (uint) (share->rec[field].huff_tree-share->decode_trees)+1, + share->rec[field].huff_tree->quick_table_bits); + } + VOID(putchar('\n')); + start+=share->rec[field].length; + } + } + DBUG_VOID_RETURN; +} /* describe */ + + + /* Sort records according to one key */ + +static int mi_sort_records(MI_CHECK *param, + register MI_INFO *info, my_string name, + uint sort_key, + my_bool write_info, + my_bool update_index) +{ + int got_error; + uint key; + MI_KEYDEF *keyinfo; + File new_file; + uchar *temp_buff; + ha_rows old_record_count; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("sort_records"); + + bzero((char*) sort_info,sizeof(sort_info)); + keyinfo= &share->keyinfo[sort_key]; + got_error=1; + temp_buff=0; + new_file= -1; + + if (!(((ulonglong) 1 << sort_key) & share->state.key_map)) + { + mi_check_print_error(param,"Can't sort table '%s' on key %d; No such key", + name,sort_key+1); + param->error_printed=0; + DBUG_RETURN(-1); + } + if (!(param->testflag & T_SILENT)) + { + printf("- Sorting records for MyISAM-table '%s'\n",name); + if (write_info) + printf("Data records: %9s Deleted: %9s\n", + llstr(info->state->records,llbuff), + llstr(info->state->del,llbuff2)); + } + if (share->state.key_root[sort_key] == HA_OFFSET_ERROR) + DBUG_RETURN(0); /* Nothing to do */ + + init_key_cache(param->use_buffers,NEAD_MEM); + if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length, + WRITE_CACHE,share->pack.header_length,1, + MYF(MY_WME | MY_WAIT_IF_FULL))) + goto err; + info->opt_flag|=WRITE_CACHE_USED; + + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not enough memory for key block"); + goto err; + } + if (!(sort_info->record=(byte*) my_malloc((uint) share->base.pack_reclength, + MYF(0)))) + { + mi_check_print_error(param,"Not enough memory for record"); + goto err; + } + new_file=my_raid_create(fn_format(param->temp_filename,name,"", + DATA_TMP_EXT,2+4), + 0,param->tmpfile_createflag, + share->base.raid_type, + share->base.raid_chunks, + share->base.raid_chunksize, + MYF(0)); + if (new_file < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (filecopy(param,new_file,info->dfile,0L,share->pack.header_length, + "datafile-header")) + goto err; + info->rec_cache.file=new_file; /* Use this file for cacheing*/ + + lock_memory(param); + for (key=0 ; key < share->base.keys ; key++) + share->keyinfo[key].flag|= HA_SORT_ALLOWS_SAME; + + if (my_pread(share->kfile,(byte*) temp_buff, + (uint) keyinfo->block_length, + share->state.key_root[sort_key], + MYF(MY_NABP+MY_WME))) + { + mi_check_print_error(param,"Can't read indexpage from filepos: %s", + (ulong) share->state.key_root[sort_key]); + goto err; + } + + /* Setup param for sort_write_record */ + sort_info->info=info; + sort_info->new_data_file_type=share->data_file_type; + sort_info->fix_datafile=1; + sort_info->filepos=share->pack.header_length; + old_record_count=info->state->records; + info->state->records=0; + if (sort_info->new_data_file_type != COMPRESSED_RECORD) + share->state.checksum=0; + + if (sort_record_index(param, info,keyinfo,share->state.key_root[sort_key], + temp_buff, sort_key,new_file,update_index) || + write_data_suffix(param, info) || + flush_io_cache(&info->rec_cache)) + goto err; + + if (info->state->records != old_record_count) + { + mi_check_print_error(param,"found %s of %s records", + llstr(info->state->records,llbuff), + llstr(old_record_count,llbuff2)); + goto err; + } + + VOID(my_close(info->dfile,MYF(MY_WME))); + param->out_flag|=O_NEW_DATA; /* Data in new file */ + info->dfile=new_file; /* Use new datafile */ + info->state->del=0; + info->state->empty=0; + share->state.dellink= HA_OFFSET_ERROR; + info->state->data_file_length=sort_info->filepos; + share->state.split=info->state->records; /* Only hole records */ + share->state.version=(ulong) time((time_t*) 0); + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (param->testflag & T_WRITE_LOOP) + { + VOID(fputs(" \r",stdout)); VOID(fflush(stdout)); + } + got_error=0; + +err: + if (got_error && new_file >= 0) + { + VOID(end_io_cache(&info->rec_cache)); + (void) my_close(new_file,MYF(MY_WME)); + (void) my_raid_delete(param->temp_filename, share->base.raid_chunksize, + MYF(MY_WME)); + } + if (temp_buff) + { + my_afree((gptr) temp_buff); + } + my_free(sort_info->record,MYF(MY_ALLOW_ZERO_PTR)); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + VOID(end_io_cache(&info->rec_cache)); + my_free(sort_info->buff,MYF(MY_ALLOW_ZERO_PTR)); + sort_info->buff=0; + share->state.sortkey=sort_key; + DBUG_RETURN(flush_blocks(param, share->kfile) | got_error); +} /* sort_records */ + + + /* Sort records recursive using one index */ + +static int sort_record_index(MI_CHECK *param,MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, uint sort_key, + File new_file,my_bool update_index) +{ + uint nod_flag,used_length,key_length; + uchar *temp_buff,*keypos,*endpos; + my_off_t next_page,rec_pos; + uchar lastkey[MI_MAX_KEY_BUFF]; + char llbuff[22]; + SORT_INFO *sort_info= ¶m->sort_info; + DBUG_ENTER("sort_record_index"); + + nod_flag=mi_test_if_nod(buff); + temp_buff=0; + + if (nod_flag) + { + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not Enough memory"); + DBUG_RETURN(-1); + } + } + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + for ( ;; ) + { + _sanity(__FILE__,__LINE__); + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + if (my_pread(info->s->kfile,(byte*) temp_buff, + (uint) keyinfo->block_length, next_page, + MYF(MY_NABP+MY_WME))) + { + mi_check_print_error(param,"Can't read keys from filepos: %s", + llstr(next_page,llbuff)); + goto err; + } + if (sort_record_index(param, info,keyinfo,next_page,temp_buff,sort_key, + new_file, update_index)) + goto err; + } + _sanity(__FILE__,__LINE__); + if (keypos >= endpos || + (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey)) + == 0) + break; + rec_pos= _mi_dpos(info,0,lastkey+key_length); + + if ((*info->s->read_rnd)(info,sort_info->record,rec_pos,0)) + { + mi_check_print_error(param,"%d when reading datafile",my_errno); + goto err; + } + if (rec_pos != sort_info->filepos && update_index) + { + _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength, + sort_info->filepos); + if (movepoint(info,sort_info->record,rec_pos,sort_info->filepos, + sort_key)) + { + mi_check_print_error(param,"%d when updating key-pointers",my_errno); + goto err; + } + } + if (sort_write_record(sort_info)) + goto err; + } + /* Clear end of block to get better compression if the table is backuped */ + bzero((byte*) buff+used_length,keyinfo->block_length-used_length); + if (my_pwrite(info->s->kfile,(byte*) buff,(uint) keyinfo->block_length, + page,param->myf_rw)) + { + mi_check_print_error(param,"%d when updating keyblock",my_errno); + goto err; + } + if (temp_buff) + my_afree((gptr) temp_buff); + DBUG_RETURN(0); +err: + if (temp_buff) + my_afree((gptr) temp_buff); + DBUG_RETURN(1); +} /* sort_record_index */ + + + /* print warnings and errors */ + /* VARARGS */ + +void mi_check_print_info(MI_CHECK *param __attribute__((unused)), + const char *fmt,...) +{ + va_list args; + + va_start(args,fmt); + VOID(vfprintf(stdout, fmt, args)); + VOID(fputc('\n',stdout)); + va_end(args); + return; +} + +/* VARARGS */ + +void mi_check_print_warning(MI_CHECK *param, const char *fmt,...) +{ + va_list args; + DBUG_ENTER("mi_check_print_warning"); + + fflush(stdout); + if (!param->warning_printed && !param->error_printed) + { + if (param->testflag & T_SILENT) + fprintf(stderr,"%s: MyISAM file %s\n",my_progname, + param->isam_file_name); + } + param->warning_printed=1; + va_start(args,fmt); + fprintf(stderr,"%s: warning: ",my_progname); + VOID(vfprintf(stderr, fmt, args)); + VOID(fputc('\n',stderr)); + fflush(stderr); + va_end(args); + DBUG_VOID_RETURN; +} + +/* VARARGS */ + +void mi_check_print_error(MI_CHECK *param, const char *fmt,...) +{ + va_list args; + DBUG_ENTER("mi_check_print_error"); + DBUG_PRINT("enter",("format: %s",fmt)); + + fflush(stdout); + if (!param->warning_printed && !param->error_printed) + { + if (param->testflag & T_SILENT) + fprintf(stderr,"%s: ISAM file %s\n",my_progname,param->isam_file_name); + } + param->error_printed|=1; + va_start(args,fmt); + fprintf(stderr,"%s: error: ",my_progname); + VOID(vfprintf(stderr, fmt, args)); + VOID(fputc('\n',stderr)); + fflush(stderr); + va_end(args); + DBUG_VOID_RETURN; +} diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h new file mode 100644 index 00000000000..13bb2e7efad --- /dev/null +++ b/myisam/myisamdef.h @@ -0,0 +1,621 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* This file is included by all internal myisam files */ + +#define ISAM_LIBRARY +#include "myisam.h" /* Structs & some defines */ +#include "myisampack.h" /* packing of keys */ +#ifdef THREAD +#include <my_pthread.h> +#include <thr_lock.h> +#else +#include <my_no_pthread.h> +#endif + +#if defined(my_write) && !defined(MAP_TO_USE_RAID) +#undef my_write /* undef map from my_nosys; We need test-if-disk full */ +#endif + +typedef struct st_mi_status_info +{ + ha_rows records; /* Rows in table */ + ha_rows del; /* Removed rows */ + my_off_t empty; /* lost space in datafile */ + my_off_t key_empty; /* lost space in indexfile */ + my_off_t key_file_length; + my_off_t data_file_length; +} MI_STATUS_INFO; + +typedef struct st_mi_state_info +{ + struct { /* Fileheader */ + uchar file_version[4]; + uchar options[2]; + uchar header_length[2]; + uchar state_info_length[2]; + uchar base_info_length[2]; + uchar base_pos[2]; + uchar key_parts[2]; /* Key parts */ + uchar unique_key_parts[2]; /* Key parts + unique parts */ + uchar keys; /* number of keys in file */ + uchar uniques; /* number of UNIQUE definitions */ + uchar language; /* Language for indexes */ + uchar max_block_size; /* max keyblock size */ + uchar not_used[2]; /* To align to 8 */ + } header; + + MI_STATUS_INFO state; + ha_rows split; /* number of split blocks */ + my_off_t dellink; /* Link to next removed block */ + ulonglong auto_increment; + ulong process; /* process that updated table last */ + ulong unique; /* Unique number for this process */ + ulong status; + my_off_t *key_root; /* Start of key trees */ + my_off_t *key_del; /* delete links for trees */ + + ulong sec_index_changed; /* Updated when new sec_index */ + ulong sec_index_used; /* which extra index are in use */ + ulonglong key_map; /* Which keys are in use */ + ha_checksum checksum; + ulong version; /* timestamp of create */ + time_t create_time; /* Time when created database */ + time_t recover_time; /* Time for last recover */ + time_t check_time; /* Time for last check */ + uint sortkey; /* sorted by this key (not used) */ + uint open_count; + bool changed; /* Changed since isamchk */ + my_off_t rec_per_key_rows; /* Rows when calculating rec_per_key */ + ulong *rec_per_key_part; + + /* the following isn't saved on disk */ + uint state_diff_length; /* Should be 0 */ + uint state_length; /* Length of state header in file */ + ulong *key_info; +} MI_STATE_INFO; + +#define MI_STATE_INFO_SIZE (24+14*8+7*4+2*2+8) +#define MI_STATE_KEY_SIZE 8 +#define MI_STATE_KEYBLOCK_SIZE 8 +#define MI_STATE_KEYSEG_SIZE 4 +#define MI_STATE_EXTRA_SIZE ((MI_MAX_KEY+MI_MAX_KEY_BLOCK_SIZE)*MI_STATE_KEY_SIZE + MI_MAX_KEY*MI_MAX_KEY_SEG*MI_STATE_KEYSEG_SIZE) +#define MI_KEYDEF_SIZE (2+ 5*2) +#define MI_UNIQUEDEF_SIZE (2+1+1) +#define MI_KEYSEG_SIZE (6+ 2*2 + 4*2) +#define MI_COLUMNDEF_SIZE (2*3+1) +#define MI_BASE_INFO_SIZE (5*8 + 8*4 + 4 + 4*2 + 16) +#define MI_INDEX_BLOCK_MARGIN 16 /* Safety margin for .MYI tables */ + +typedef struct st_mi_base_info +{ + my_off_t keystart; /* Start of keys */ + my_off_t max_data_file_length; + my_off_t max_key_file_length; + my_off_t margin_key_file_length; + ha_rows records,reloc; /* Create information */ + ulong mean_row_length; /* Create information */ + ulong reclength; /* length of unpacked record */ + ulong pack_reclength; /* Length of full packed rec. */ + ulong min_pack_length; + ulong max_pack_length; /* Max possibly length of packed rec.*/ + ulong min_block_length; + ulong fields, /* fields in table */ + pack_fields; /* packed fields in table */ + uint rec_reflength; /* = 2-8 */ + uint key_reflength; /* = 2-8 */ + uint keys; /* same as in state.header */ + uint auto_key; /* Which key-1 is a auto key */ + uint blobs; /* Number of blobs */ + uint pack_bits; /* Length of packed bits */ + uint max_key_block_length; /* Max block length */ + uint max_key_length; /* Max key length */ + /* Extra allocation when using dynamic record format */ + uint extra_alloc_bytes; + uint extra_alloc_procent; + /* Info about raid */ + uint raid_type,raid_chunks; + ulong raid_chunksize; + /* The following are from the header */ + uint key_parts,all_key_parts; +} MI_BASE_INFO; + + + /* Structs used intern in database */ + +typedef struct st_mi_blob /* Info of record */ +{ + ulong offset; /* Offset to blob in record */ + uint pack_length; /* Type of packed length */ + ulong length; /* Calc:ed for each record */ +} MI_BLOB; + + +typedef struct st_mi_isam_pack { + ulong header_length; + uint ref_length; +} MI_PACK; + + +typedef struct st_mi_isam_share { /* Shared between opens */ + MI_STATE_INFO state; + MI_BASE_INFO base; + MI_KEYDEF *keyinfo; /* Key definitions */ + MI_UNIQUEDEF *uniqueinfo; /* unique definitions */ + MI_KEYSEG *keyparts; /* key part info */ + MI_COLUMNDEF *rec; /* Pointer to field information */ + MI_PACK pack; /* Data about packed records */ + MI_BLOB *blobs; /* Pointer to blobs */ + char *filename; /* Name of indexfile */ + byte *file_map; /* mem-map of file if possible */ + ulong this_process; /* processid */ + ulong last_process; /* For table-change-check */ + ulong last_version; /* Version on start */ + ulong options; /* Options used */ + uint rec_reflength; /* rec_reflength in use now */ + int kfile; /* Shared keyfile */ + int data_file; /* Shared data file */ + int mode; /* mode of file on open */ + uint reopen; /* How many times reopened */ + uint w_locks,r_locks; /* Number of read/write locks */ + uint blocksize; /* blocksize of keyfile */ + ulong min_pack_length; /* Theese are used by packed data */ + ulong max_pack_length; + ulong state_diff_length; + my_bool changed, /* If changed since lock */ + global_changed, /* If changed since open */ + not_flushed, + temporary,delay_key_write, + concurrent_insert; + myf write_flag; + int rnd; /* rnd-counter */ + MI_DECODE_TREE *decode_trees; + uint16 *decode_tables; + enum data_file_type data_file_type; + int (*read_record)(struct st_myisam_info*, my_off_t, byte*); + int (*write_record)(struct st_myisam_info*, const byte*); + int (*update_record)(struct st_myisam_info*, my_off_t, const byte*); + int (*delete_record)(struct st_myisam_info*); + int (*read_rnd)(struct st_myisam_info*, byte*, my_off_t, my_bool); + int (*compare_record)(struct st_myisam_info*, const byte *); + ha_checksum (*calc_checksum)(struct st_myisam_info*, const byte *); + int (*compare_unique)(struct st_myisam_info*, MI_UNIQUEDEF *, + const byte *record, my_off_t pos); +#ifdef THREAD + THR_LOCK lock; + pthread_mutex_t intern_lock; /* Locking for use with _locking */ + rw_lock_t *key_root_lock; +#endif +} MYISAM_SHARE; + + +typedef uint mi_bit_type; + +typedef struct st_mi_bit_buff { /* Used for packing of record */ + mi_bit_type current_byte; + uint bits; + uchar *pos,*end,*blob_pos; + uint error; +} MI_BIT_BUFF; + + +struct st_myisam_info { + MYISAM_SHARE *s; /* Shared between open:s */ + MI_STATUS_INFO *state,save_state; + MI_BLOB *blobs; /* Pointer to blobs */ + int dfile; /* The datafile */ + MI_BIT_BUFF bit_buff; + uint opt_flag; /* Optim. for space/speed */ + uint update; /* If file changed since open */ + char *filename; /* parameter to open filename */ + ulong this_unique; /* uniq filenumber or thread */ + ulong last_unique; /* last unique number */ + ulong this_loop; /* counter for this open */ + ulong last_loop; /* last used counter */ + my_off_t lastpos, /* Last record position */ + nextpos; /* Position to next record */ + my_off_t save_lastpos; + my_off_t pos; /* Intern variable */ + ha_checksum checksum; + ulong packed_length,blob_length; /* Length of found, packed record */ + uint alloced_rec_buff_length; /* Max recordlength malloced */ + uchar *buff, /* Temp area for key */ + *lastkey,*lastkey2; /* Last used search key */ + byte *rec_buff, /* Tempbuff for recordpack */ + *rec_alloc; /* Malloced area for record */ + uchar *int_keypos, /* Save position for next/previous */ + *int_maxpos; /* -""- */ + uint32 int_keytree_version; /* -""- */ + uint int_nod_flag; /* -""- */ + my_off_t last_keypage; /* Last key page read */ + my_off_t last_search_keypage; /* Last keypage when searching */ + my_off_t dupp_key_pos; + int lastinx; /* Last used index */ + uint lastkey_length; /* Length of key in lastkey */ + uint last_rkey_length; /* Last length in mi_rkey() */ + uint save_lastkey_length; + int errkey; /* Got last error on this key */ + int lock_type; /* How database was locked */ + int tmp_lock_type; /* When locked by readinfo */ + uint data_changed; /* Somebody has changed data */ + uint save_update; /* When using KEY_READ */ + int save_lastinx; + my_bool was_locked; /* Was locked in panic */ + my_bool quick_mode; + my_bool page_changed; /* If info->buff can't be used for rnext */ + my_bool buff_used; /* If info->buff has to be reread for rnext */ + myf lock_wait; /* is 0 or MY_DONT_WAIT */ + int (*read_record)(struct st_myisam_info*, my_off_t, byte*); + LIST open_list; + IO_CACHE rec_cache; /* When cacheing records */ +#ifdef THREAD + THR_LOCK_DATA lock; +#endif +}; + + + /* Some defines used by isam-funktions */ + +#define USE_WHOLE_KEY MI_MAX_KEY_BUFF*2 /* Use whole key in _mi_search() */ +#define F_EXTRA_LCK -1 + + /* bits in opt_flag */ +#define MEMMAP_USED 32 +#define REMEMBER_OLD_POS 64 + +#define WRITEINFO_UPDATE_KEYFILE 1 +#define WRITEINFO_NO_UNLOCK 2 + +#define mi_getint(x) ((uint) mi_uint2korr(x) & 32767) +#define mi_putint(x,y,nod) { uint16 boh=(nod ? (uint16) 32768 : 0) + (uint16) (y);\ + mi_int2store(x,boh); } +#define mi_test_if_nod(x) (x[0] & 128 ? info->s->base.key_reflength : 0) +#define mi_mark_crashed(x) (x)->s->state.changed|=2 +#define mi_is_crashed(x) ((x)->s->state.changed & 2) + +/* Functions to store length of space packed keys, VARCHAR or BLOB keys */ + +#define store_key_length_inc(key,length) \ +{ if ((length) < 255) \ + { *(key)++=(length); } \ + else \ + { *(key)=255; mi_int2store((key)+1,(length)); (key)+=3; } \ +} + +#define store_key_length(key,length) \ +{ if ((length) < 255) \ + { *(key)=(length); } \ + else \ + { *(key)=255; mi_int2store((key)+1,(length)); } \ +} + +#define get_key_length(length,key) \ +{ if ((uchar) *(key) != 255) \ + length= (uint) (uchar) *((key)++); \ + else \ + { length=mi_uint2korr((key)+1); (key)+=3; } \ +} + +#define get_key_full_length(length,key) \ +{ if ((uchar) *(key) != 255) \ + length= ((uint) (uchar) *((key)++))+1; \ + else \ + { length=mi_uint2korr((key)+1)+3; (key)+=3; } \ +} + +#define get_key_pack_length(length,length_pack,key) \ +{ if ((uchar) *(key) != 255) \ + { length= (uint) (uchar) *((key)++); length_pack=1; }\ + else \ + { length=mi_uint2korr((key)+1); (key)+=3; length_pack=3; } \ +} + +#define get_pack_length(length) ((length) >= 255 ? 3 : 1) + +#define MI_MIN_BLOCK_LENGTH 20 /* Because of delete-link */ +#define MI_EXTEND_BLOCK_LENGTH 20 /* Don't use to small record-blocks */ +#define MI_SPLIT_LENGTH ((MI_EXTEND_BLOCK_LENGTH+4)*2) +#define MI_MAX_DYN_BLOCK_HEADER 20 /* Max prefix of record-block */ +#define MI_BLOCK_INFO_HEADER_LENGTH 20 +#define MI_DYN_DELETE_BLOCK_HEADER 20 /* length of delete-block-header */ +#define MI_DYN_MAX_BLOCK_LENGTH ((1L << 24)-4L) +#define MI_DYN_MAX_ROW_LENGTH (MI_DYN_MAX_BLOCK_LENGTH - MI_SPLIT_LENGTH) +#define MI_DYN_ALIGN_SIZE 4 /* Align blocks on this */ +#define MI_MAX_DYN_HEADER_BYTE 12 /* max header byte for dynamic rows */ + +#define MEMMAP_EXTRA_MARGIN 7 /* Write this as a suffix for file */ + +#define PACK_TYPE_SELECTED 1 /* Bits in field->pack_type */ +#define PACK_TYPE_SPACE_FIELDS 2 +#define PACK_TYPE_ZERO_FILL 4 +#define MI_FOUND_WRONG_KEY 32738 /* Impossible value from _mi_key_cmp */ + +#define MI_KEY_BLOCK_LENGTH 1024 /* Min key block length */ +#define MI_MAX_KEY_BLOCK_LENGTH 8192 +#define MI_MAX_KEY_BLOCK_SIZE (MI_MAX_KEY_BLOCK_LENGTH/MI_KEY_BLOCK_LENGTH) +#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer) ((((key_length+data_pointer+key_pointer)*4+key_pointer+2)/MI_KEY_BLOCK_LENGTH+1)*MI_KEY_BLOCK_LENGTH) +#define MI_MAX_KEYPTR_SIZE 5 /* For calculating block lengths */ + +/* The UNIQUE check is done with a hashed long key */ + +#define MI_UNIQUE_HASH_TYPE HA_KEYTYPE_ULONG_INT +#define mi_unique_store(A,B) mi_int4store((A),(B)) + +#ifdef THREAD +extern pthread_mutex_t THR_LOCK_myisam; +#endif +#if !defined(THREAD) || defined(DONT_USE_RW_LOCKS) +#define rw_wrlock(A) {} +#define rw_rdlock(A) {} +#define rw_unlock(A) {} +#endif + + /* Some extern variables */ + +extern LIST *myisam_open_list; +extern uchar NEAR myisam_file_magic[],NEAR myisam_pack_file_magic[]; +extern uint NEAR myisam_read_vec[],NEAR myisam_readnext_vec[]; +extern uint myisam_quick_table_bits; +extern File myisam_log_file; + + /* This is used by _mi_calc_xxx_key_length och _mi_store_key */ + +typedef struct st_mi_s_param +{ + uint ref_length,key_length, + n_ref_length, + n_length, + totlength, + part_of_prev_key,prev_length,pack_marker; + uchar *key, *prev_key,*next_key_pos; + bool store_not_null; +} MI_KEY_PARAM; + + /* Prototypes for intern functions */ + +extern int _mi_read_dynamic_record(MI_INFO *info,my_off_t filepos,byte *buf); +extern int _mi_write_dynamic_record(MI_INFO*, const byte*); +extern int _mi_update_dynamic_record(MI_INFO*, my_off_t, const byte*); +extern int _mi_delete_dynamic_record(MI_INFO *info); +extern int _mi_cmp_dynamic_record(MI_INFO *info,const byte *record); +extern int _mi_read_rnd_dynamic_record(MI_INFO *, byte *,my_off_t, my_bool); +extern int _mi_write_blob_record(MI_INFO*, const byte*); +extern int _mi_update_blob_record(MI_INFO*, my_off_t, const byte*); +extern int _mi_read_static_record(MI_INFO *info, my_off_t filepos,byte *buf); +extern int _mi_write_static_record(MI_INFO*, const byte*); +extern int _mi_update_static_record(MI_INFO*, my_off_t, const byte*); +extern int _mi_delete_static_record(MI_INFO *info); +extern int _mi_cmp_static_record(MI_INFO *info,const byte *record); +extern int _mi_read_rnd_static_record(MI_INFO*, byte *,my_off_t, my_bool); +extern int _mi_ck_write(MI_INFO *info,uint keynr,uchar *key,uint length); +extern int _mi_enlarge_root(MI_INFO *info,uint keynr,uchar *key); +extern int _mi_insert(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uchar *anc_buff,uchar *key_pos,uchar *key_buff, + uchar *father_buff, uchar *father_keypos, + my_off_t father_page, my_bool insert_last); +extern int _mi_split_page(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uchar *buff,uchar *key_buff, my_bool insert_last); +extern uchar *_mi_find_half_pos(uint nod_flag,MI_KEYDEF *keyinfo,uchar *page, + uchar *key,uint *return_key_length, + uchar **after_key); +extern int _mi_calc_static_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *key_buff, + uchar *key, MI_KEY_PARAM *s_temp); +extern int _mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *key_buff, + uchar *key, MI_KEY_PARAM *s_temp); +extern int _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *prev_key, + uchar *key, MI_KEY_PARAM *s_temp); +extern int _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *key_pos,uchar *org_key, + uchar *prev_key, + uchar *key, MI_KEY_PARAM *s_temp); +void _mi_store_static_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); +void _mi_store_var_pack_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); +#ifdef NOT_USED +void _mi_store_pack_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); +#endif +void _mi_store_bin_pack_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); + +extern int _mi_ck_delete(MI_INFO *info,uint keynr,uchar *key,uint key_length); +extern int _mi_readinfo(MI_INFO *info,int lock_flag,int check_keybuffer); +extern int _mi_writeinfo(MI_INFO *info,uint options); +extern int _mi_test_if_changed(MI_INFO *info); +extern int _mi_mark_file_changed(MI_INFO *info); +extern int _mi_decrement_open_count(MI_INFO *info); +extern int _mi_check_index(MI_INFO *info,int inx); +extern int _mi_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,uint key_len, + uint nextflag,my_off_t pos); +extern int _mi_bin_search(struct st_myisam_info *info,MI_KEYDEF *keyinfo, + uchar *page,uchar *key,uint key_len,uint comp_flag, + uchar * *ret_pos,uchar *buff, my_bool *was_last_key); +extern int _mi_seq_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *page, + uchar *key,uint key_len,uint comp_flag, + uchar **ret_pos,uchar *buff, my_bool *was_last_key); +extern int _mi_compare_text(CHARSET_INFO *, uchar *, uint, uchar *, uint , + my_bool); +extern my_off_t _mi_kpos(uint nod_flag,uchar *after_key); +extern void _mi_kpointer(MI_INFO *info,uchar *buff,my_off_t pos); +extern my_off_t _mi_dpos(MI_INFO *info, uint nod_flag,uchar *after_key); +extern my_off_t _mi_rec_pos(MYISAM_SHARE *info, uchar *ptr); +extern void _mi_dpointer(MI_INFO *info, uchar *buff,my_off_t pos); +extern int _mi_key_cmp(MI_KEYSEG *keyseg, uchar *a,uchar *b, + uint key_length,uint nextflag,uint *diff_length); +extern uint _mi_get_static_key(MI_KEYDEF *keyinfo,uint nod_flag,uchar * *page, + uchar *key); +extern uint _mi_get_pack_key(MI_KEYDEF *keyinfo,uint nod_flag,uchar * *page, + uchar *key); +extern uint _mi_get_binary_pack_key(MI_KEYDEF *keyinfo, uint nod_flag, + uchar **page_pos, uchar *key); +extern uchar *_mi_get_last_key(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *keypos, + uchar *lastkey,uchar *endpos, + uint *return_key_length); +extern uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, uint *return_key_length); +extern uint _mi_keylength(MI_KEYDEF *keyinfo,uchar *key); +extern uchar *_mi_move_key(MI_KEYDEF *keyinfo,uchar *to,uchar *from); +extern int _mi_search_next(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uint key_length,uint nextflag,my_off_t pos); +extern int _mi_search_first(MI_INFO *info,MI_KEYDEF *keyinfo,my_off_t pos); +extern int _mi_search_last(MI_INFO *info,MI_KEYDEF *keyinfo,my_off_t pos); +extern uchar *_mi_fetch_keypage(MI_INFO *info,MI_KEYDEF *keyinfo,my_off_t page, + uchar *buff,int return_buffer); +extern int _mi_write_keypage(MI_INFO *info,MI_KEYDEF *keyinfo,my_off_t page, + uchar *buff); +extern int _mi_dispose(MI_INFO *info,MI_KEYDEF *keyinfo,my_off_t pos); +extern my_off_t _mi_new(MI_INFO *info,MI_KEYDEF *keyinfo); +extern uint _mi_make_key(MI_INFO *info,uint keynr,uchar *key, + const byte *record,my_off_t filepos); +extern uint _mi_pack_key(MI_INFO *info,uint keynr,uchar *key,uchar *old, + uint key_length); +extern int _mi_read_key_record(MI_INFO *info,my_off_t filepos,byte *buf); +extern int _mi_read_cache(IO_CACHE *info,byte *buff,my_off_t pos, + uint length,int re_read_if_possibly); +extern void update_auto_increment(MI_INFO *info,const byte *record); +extern byte *mi_fix_rec_buff_for_blob(MI_INFO *info,ulong blob_length); +extern ulong _mi_rec_unpack(MI_INFO *info,byte *to,byte *from, + ulong reclength); +extern my_bool _mi_rec_check(MI_INFO *info,const char *from); +extern int _mi_write_part_record(MI_INFO *info,my_off_t filepos,ulong length, + my_off_t next_filepos,byte **record, + ulong *reclength,int *flag); +extern void _mi_print_key(FILE *stream,MI_KEYSEG *keyseg,const uchar *key, + uint length); +extern my_bool _mi_read_pack_info(MI_INFO *info,pbool fix_keys); +extern int _mi_read_pack_record(MI_INFO *info,my_off_t filepos,byte *buf); +extern int _mi_read_rnd_pack_record(MI_INFO*, byte *,my_off_t, my_bool); +extern int _mi_pack_rec_unpack(MI_INFO *info,byte *to,byte *from, + ulong reclength); +extern ulonglong mi_safe_mul(ulonglong a,ulonglong b); + +struct st_sort_info; + + +typedef struct st_mi_block_info { /* Parameter to _mi_get_block_info */ + uchar header[MI_BLOCK_INFO_HEADER_LENGTH]; + ulong rec_len; + ulong data_len; + ulong block_len; + ulong blob_len; + my_off_t filepos; + my_off_t next_filepos; + my_off_t prev_filepos; + uint second_read; + uint offset; +} MI_BLOCK_INFO; + + /* bits in return from _mi_get_block_info */ + +#define BLOCK_FIRST 1 +#define BLOCK_LAST 2 +#define BLOCK_DELETED 4 +#define BLOCK_ERROR 8 /* Wrong data */ +#define BLOCK_SYNC_ERROR 16 /* Right data at wrong place */ +#define BLOCK_FATAL_ERROR 32 /* hardware-error */ + +#define NEAD_MEM ((uint) 10*4*(IO_SIZE+32)+32) /* Nead for recursion */ +#define MAXERR 20 +#define BUFFERS_WHEN_SORTING 16 /* Alloc for sort-key-tree */ +#define WRITE_COUNT MY_HOW_OFTEN_TO_WRITE +#define INDEX_TMP_EXT ".TMM" +#define DATA_TMP_EXT ".TMD" + +#define UPDATE_TIME 1 +#define UPDATE_STAT 2 +#define UPDATE_SORT 4 +#define UPDATE_AUTO_INC 8 +#define UPDATE_OPEN_COUNT 16 + +#define USE_BUFFER_INIT (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE) +#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD) +#define SORT_BUFFER_INIT (2048L*1024L-MALLOC_OVERHEAD) +#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD) + +enum myisam_log_commands { + MI_LOG_OPEN,MI_LOG_WRITE,MI_LOG_UPDATE,MI_LOG_DELETE,MI_LOG_CLOSE,MI_LOG_EXTRA,MI_LOG_LOCK,MI_LOG_DELETE_ALL +}; + +#define myisam_log(a,b,c,d) if (myisam_log_file >= 0) _myisam_log(a,b,c,d) +#define myisam_log_command(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_command(a,b,c,d,e) +#define myisam_log_record(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_record(a,b,c,d,e) + +extern uint _mi_get_block_info(MI_BLOCK_INFO *,File, my_off_t); +extern uint _mi_rec_pack(MI_INFO *info,byte *to,const byte *from); +extern uint _mi_pack_get_block_info(MI_INFO *mysql, MI_BLOCK_INFO *, File, + my_off_t, char *rec_buf); +extern void _my_store_blob_length(byte *pos,uint pack_length,uint length); +extern void _myisam_log(enum myisam_log_commands command,MI_INFO *info, + const byte *buffert,uint length); +extern void _myisam_log_command(enum myisam_log_commands command, + MI_INFO *info, const byte *buffert, + uint length, int result); +extern void _myisam_log_record(enum myisam_log_commands command,MI_INFO *info, + const byte *record,my_off_t filepos, + int result); +extern my_bool _mi_memmap_file(MI_INFO *info); +extern void _mi_unmap_file(MI_INFO *info); +extern uint save_pack_length(byte *block_buff,ulong length); + +uint mi_state_info_write(File file, MI_STATE_INFO *state, uint pWrite); +char *mi_state_info_read(char *ptr, MI_STATE_INFO *state); +uint mi_state_info_read_dsk(File file, MI_STATE_INFO *state, my_bool pRead); +uint mi_base_info_write(File file, MI_BASE_INFO *base); +char *my_n_base_info_read(char *ptr, MI_BASE_INFO *base); +int mi_keyseg_write(File file, const MI_KEYSEG *keyseg); +char *mi_keyseg_read(char *ptr, MI_KEYSEG *keyseg); +uint mi_keydef_write(File file, MI_KEYDEF *keydef); +char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef); +uint mi_uniquedef_write(File file, MI_UNIQUEDEF *keydef); +char *mi_uniquedef_read(char *ptr, MI_UNIQUEDEF *keydef); +uint mi_recinfo_write(File file, MI_COLUMNDEF *recinfo); +char *mi_recinfo_read(char *ptr, MI_COLUMNDEF *recinfo); +ulong _my_calc_total_blob_length(MI_INFO *info, const byte *record); +ha_checksum mi_checksum(MI_INFO *info, const byte *buf); +ha_checksum mi_static_checksum(MI_INFO *info, const byte *buf); +my_bool mi_check_unique(MI_INFO *info, MI_UNIQUEDEF *def, byte *record, + ha_checksum unique_hash, my_off_t pos); +ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *buf); +int _mi_cmp_static_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const byte *record, my_off_t pos); +int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const byte *record, my_off_t pos); +int mi_unique_comp(MI_UNIQUEDEF *def, const byte *a, const byte *b, + my_bool null_are_equal); +void mi_get_status(void* param); +void mi_update_status(void* param); +void mi_copy_status(void* to,void *from); +my_bool mi_check_status(void* param); + +/* Functions needed by mi_check */ +#ifdef __cplusplus +extern "C" { +#endif +void mi_check_print_error _VARARGS((MI_CHECK *param, const char *fmt,...)); +void mi_check_print_warning _VARARGS((MI_CHECK *param, const char *fmt,...)); +void mi_check_print_info _VARARGS((MI_CHECK *param, const char *fmt,...)); + +#ifdef __cplusplus +} +#endif + diff --git a/myisam/myisamlog.c b/myisam/myisamlog.c new file mode 100644 index 00000000000..c55aecfdfa6 --- /dev/null +++ b/myisam/myisamlog.c @@ -0,0 +1,832 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* write whats in isam.log */ + +#ifndef USE_MY_FUNC +#define USE_MY_FUNC +#endif + +#include "myisamdef.h" +#include "my_tree.h" +#include <stdarg.h> +#ifdef HAVE_GETRUSAGE +#include <sys/resource.h> +#endif + +#define FILENAME(A) (A ? A->show_name : "Unknown") + +struct file_info { + long process; + int filenr,id; + my_string name,show_name,record; + MI_INFO *isam; + bool closed,used; + ulong accessed; +}; + +struct test_if_open_param { + my_string name; + int max_id; +}; + +struct st_access_param +{ + ulong min_accessed; + struct file_info *found; +}; + +#define NO_FILEPOS (ulong) ~0L + +extern int main(int argc,char * *argv); +static void get_options(int *argc,char ***argv); +static int examine_log(my_string file_name,char **table_names); +static int read_string(IO_CACHE *file,gptr *to,uint length); +static int file_info_compare(void *a,void *b); +static int test_if_open(struct file_info *key,element_count count, + struct test_if_open_param *param); +static void fix_blob_pointers(MI_INFO *isam,byte *record); +static uint set_maximum_open_files(uint); +static int test_when_accessed(struct file_info *key,element_count count, + struct st_access_param *access_param); +static void file_info_free(struct file_info *info); +static int close_some_file(TREE *tree); +static int reopen_closed_file(TREE *tree,struct file_info *file_info); +static int find_record_with_key(struct file_info *file_info,byte *record); +static void printf_log(const char *str,...); +static bool cmp_filename(struct file_info *file_info,my_string name); + +static uint verbose=0,update=0,test_info=0,max_files=0,re_open_count=0, + recover=0,prefix_remove=0; +static my_string log_filename=0,filepath=0,write_filename=0,record_pos_file=0; +static ulong com_count[10][3],number_of_commands=(ulong) ~0L, + isamlog_process; +static my_off_t isamlog_filepos,start_offset=0,record_pos= HA_OFFSET_ERROR; +static const char *command_name[]= +{"open","write","update","delete","close","extra","lock","re-open","delete-all", NullS}; + + +int main(int argc, char **argv) +{ + int error,i,first; + ulong total_count,total_error,total_recover; + MY_INIT(argv[0]); + + log_filename=myisam_log_filename; + get_options(&argc,&argv); + /* Nr of isam-files */ + max_files=(set_maximum_open_files(min(max_files,8))-6)/2; + + if (update) + printf("Trying to %s isamfiles according to log '%s'\n", + (recover ? "recover" : "update"),log_filename); + error= examine_log(log_filename,argv); + if (update && ! error) + puts("isamfile:s updated successfully"); + total_count=total_error=total_recover=0; + for (i=first=0 ; command_name[i] ; i++) + { + if (com_count[i][0]) + { + if (!first++) + { + if (verbose || update) + puts(""); + puts("Commands Used count Errors Recover errors"); + } + printf("%-12s%9ld%10ld%17ld\n",command_name[i],com_count[i][0], + com_count[i][1],com_count[i][2]); + total_count+=com_count[i][0]; + total_error+=com_count[i][1]; + total_recover+=com_count[i][2]; + } + } + if (total_count) + printf("%-12s%9ld%10ld%17ld\n","Total",total_count,total_error, + total_recover); + if (re_open_count) + printf("Had to do %d re-open because of too few possibly open files\n", + re_open_count); + VOID(mi_panic(HA_PANIC_CLOSE)); + my_end(test_info ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + exit(error); + return 0; /* No compiler warning */ +} /* main */ + + +static void get_options(register int *argc, register char ***argv) +{ + int help,version; + const char *pos,*usage; + char option; + + help=0; + usage="Usage: %s [-?iruvIV] [-c #] [-f #] [-F filepath/] [-o #] [-R file recordpos] [-w write_file] [log-filename [table ...]] \n"; + pos=""; + + while (--*argc > 0 && *(pos = *(++*argv)) == '-' ) { + while (*++pos) + { + version=0; + switch((option=*pos)) { + case '#': + DBUG_PUSH (++pos); + pos=" "; /* Skipp rest of arg */ + break; + case 'c': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + number_of_commands=(ulong) atol(pos); + pos=" "; + break; + case 'u': + update=1; + break; + case 'f': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + max_files=(uint) atoi(pos); + pos=" "; + break; + case 'i': + test_info=1; + break; + case 'o': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + start_offset=(my_off_t) strtoll(pos,NULL,10); + pos=" "; + break; + case 'p': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + prefix_remove=atoi(pos); + break; + case 'r': + update=1; + recover++; + break; + case 'R': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + record_pos_file=(char*) pos; + if (!--*argc) + goto err; + record_pos=(my_off_t) strtoll(*(++*argv),NULL,10); + pos=" "; + break; + case 'v': + verbose++; + break; + case 'w': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + write_filename=(char*) pos; + pos=" "; + break; + case 'F': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + filepath= (char*) pos; + pos=" "; + break; + case 'V': + version=1; + /* Fall through */ + case 'I': + case '?': + printf("%s Ver 1.1 for %s at %s\n",my_progname,SYSTEM_TYPE, + MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + if (version) + break; + puts("Write info about whats in a myisam log file."); + printf("If no file name is given %s is used\n",log_filename); + puts(""); + printf(usage,my_progname); + puts(""); + puts("Options: -? or -I \"Info\" -V \"version\" -c \"do only # commands\""); + puts(" -f \"max open files\" -F \"filepath\" -i \"extra info\""); + puts(" -o \"offset\" -p # \"remove # components from path\""); + puts(" -r \"recover\" -R \"file recordposition\""); + puts(" -u \"update\" -v \"verbose\" -w \"write file\""); + puts("\nOne can give a second and a third '-v' for more verbose."); + puts("Normaly one does a update (-u)."); + puts("If a recover is done all writes and all possibly updates and deletes is done\nand errors are only counted."); + puts("If one gives table names as arguments only these tables will be updated\n"); + help=1; + break; + default: + printf("illegal option: \"-%c\"\n",*pos); + break; + } + } + } + if (! *argc) + { + if (help) + exit(0); + (*argv)++; + } + if (*argc >= 1) + { + log_filename=(char*) pos; + (*argc)--; + (*argv)++; + } + return; + err: + VOID(fprintf(stderr,"option \"%c\" used without or with wrong argument\n", + option)); + exit(1); +} + + +static int examine_log(my_string file_name, char **table_names) +{ + uint command,result,files_open; + ulong access_time,length; + my_off_t filepos; + int lock_command,mi_result; + char isam_file_name[FN_REFLEN],llbuff[21],llbuff2[21]; + uchar head[20]; + gptr buff; + struct test_if_open_param open_param; + IO_CACHE cache; + File file; + FILE *write_file; + enum ha_extra_function extra_command; + TREE tree; + struct file_info file_info,*curr_file_info; + DBUG_ENTER("examine_log"); + + if ((file=my_open(file_name,O_RDONLY,MYF(MY_WME))) < 0) + DBUG_RETURN(1); + write_file=0; + if (write_filename) + { + if (!(write_file=my_fopen(write_filename,O_WRONLY,MYF(MY_WME)))) + { + my_close(file,MYF(0)); + DBUG_RETURN(1); + } + } + + init_io_cache(&cache,file,0,READ_CACHE,start_offset,0,MYF(0)); + bzero((gptr) com_count,sizeof(com_count)); + init_tree(&tree,0,sizeof(file_info),(qsort_cmp) file_info_compare,0, + (void(*)(void*)) file_info_free); + VOID(init_key_cache(KEY_CACHE_SIZE,(uint) (10*4*(IO_SIZE+MALLOC_OVERHEAD)))); + + files_open=0; access_time=0; + while (access_time++ != number_of_commands && + !my_b_read(&cache,(byte*) head,9)) + { + isamlog_filepos=my_b_tell(&cache)-9L; + file_info.filenr= mi_uint2korr(head+1); + isamlog_process=file_info.process=(long) mi_uint4korr(head+3); + result= mi_uint2korr(head+7); + if ((curr_file_info=(struct file_info*) tree_search(&tree,&file_info))) + { + curr_file_info->accessed=access_time; + if (update && curr_file_info->used && curr_file_info->closed) + { + if (reopen_closed_file(&tree,curr_file_info)) + { + command=sizeof(com_count)/sizeof(com_count[0][0])/3; + result=0; + goto com_err; + } + } + } + command=(uint) head[0]; + if (command < sizeof(com_count)/sizeof(com_count[0][0])/3 && + (!curr_file_info || curr_file_info->used)) + { + com_count[command][0]++; + if (result) + com_count[command][1]++; + } + switch ((enum myisam_log_commands) command) { + case MI_LOG_OPEN: + com_count[command][0]--; /* Must be counted explicite */ + if (result) + com_count[command][1]--; + + if (curr_file_info) + printf("\nWarning: %s is opened twice with same process and filenumber\n", + curr_file_info->show_name); + if (my_b_read(&cache,(byte*) head,2)) + goto err; + file_info.name=0; + file_info.show_name=0; + file_info.record=0; + if (read_string(&cache,(gptr*) &file_info.name, + (uint) mi_uint2korr(head))) + goto err; + { + uint i; + char *pos=file_info.name,*to; + for (i=0 ; i < prefix_remove ; i++) + { + char *next; + if (!(next=strchr(pos,FN_LIBCHAR))) + break; + pos=next+1; + } + to=isam_file_name; + if (filepath) + { + strmov(isam_file_name,filepath); + convert_dirname(isam_file_name); + to=strend(isam_file_name); + } + strmov(to,pos); + fn_ext(isam_file_name)[0]=0; /* Remove extension */ + } + open_param.name=file_info.name; + open_param.max_id=0; + VOID(tree_walk(&tree,(tree_walk_action) test_if_open,(void*) &open_param, + left_root_right)); + file_info.id=open_param.max_id+1; + file_info.show_name=my_memdup(isam_file_name, + (uint) strlen(isam_file_name)+6, + MYF(MY_WME)); + if (file_info.id > 1) + sprintf(strend(file_info.show_name),"<%d>",file_info.id); + file_info.closed=1; + file_info.accessed=access_time; + file_info.used=1; + if (table_names[0]) + { + char **name; + file_info.used=0; + for (name=table_names ; *name ; name++) + { + if (!strcmp(*name,isam_file_name)) + file_info.used=1; /* Update/log only this */ + } + } + if (update && file_info.used) + { + if (files_open >= max_files) + { + if (close_some_file(&tree)) + goto com_err; + files_open--; + } + if (!(file_info.isam= mi_open(isam_file_name,O_RDWR, + HA_OPEN_WAIT_IF_LOCKED))) + goto com_err; + if (!(file_info.record=my_malloc(file_info.isam->s->base.reclength, + MYF(MY_WME)))) + goto end; + files_open++; + file_info.closed=0; + } + VOID(tree_insert(&tree,(gptr) &file_info,0)); + if (file_info.used) + { + if (verbose && !record_pos_file) + printf_log("%s: open",file_info.show_name); + com_count[command][0]++; + if (result) + com_count[command][1]++; + } + break; + case MI_LOG_CLOSE: + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s -> %d",FILENAME(curr_file_info), + command_name[command],result); + if (curr_file_info) + { + if (!curr_file_info->closed) + files_open--; + VOID(tree_delete(&tree,(gptr) curr_file_info)); + } + break; + case MI_LOG_EXTRA: + if (my_b_read(&cache,(byte*) head,1)) + goto err; + extra_command=(enum ha_extra_function) head[0]; + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s(%d) -> %d",FILENAME(curr_file_info), + command_name[command], (int) extra_command,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_extra(curr_file_info->isam, + (int) extra_command) != (int) result) + { + VOID(fprintf(stderr, + "Warning: error %d, expected %d on command %s at %s\n", + my_errno,result,command_name[command], + llstr(isamlog_filepos,llbuff))); + } + } + break; + case MI_LOG_DELETE: + if (my_b_read(&cache,(byte*) head,8)) + goto err; + filepos=mi_sizekorr(head); + if (verbose && (!record_pos_file || + ((record_pos == filepos || record_pos == NO_FILEPOS) && + !cmp_filename(curr_file_info,record_pos_file))) && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s at %ld -> %d",FILENAME(curr_file_info), + command_name[command],(long) filepos,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_rrnd(curr_file_info->isam,curr_file_info->record,filepos)) + { + if (!recover) + goto com_err; + com_count[command][2]++; /* Mark error */ + } + mi_result=mi_delete(curr_file_info->isam,curr_file_info->record); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (mi_result) + com_count[command][2]++; /* Mark error */ + } + } + break; + case MI_LOG_WRITE: + case MI_LOG_UPDATE: + if (my_b_read(&cache,(byte*) head,12)) + goto err; + filepos=mi_sizekorr(head); + length=mi_uint4korr(head+8); + buff=0; + if (read_string(&cache,&buff,(uint) length)) + goto err; + if ((!record_pos_file || + ((record_pos == filepos || record_pos == NO_FILEPOS) && + !cmp_filename(curr_file_info,record_pos_file))) && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + { + if (write_file && + (my_fwrite(write_file,buff,length,MYF(MY_WAIT_IF_FULL | MY_NABP)))) + goto end; + if (verbose) + printf_log("%s: %s at %ld, length=%ld -> %d", + FILENAME(curr_file_info), + command_name[command], filepos,length,result); + } + if (update && curr_file_info && !curr_file_info->closed) + { + if (curr_file_info->isam->s->base.blobs) + fix_blob_pointers(curr_file_info->isam,buff); + if ((enum myisam_log_commands) command == MI_LOG_UPDATE) + { + if (mi_rrnd(curr_file_info->isam,curr_file_info->record,filepos)) + { + if (!recover) + { + result=0; + goto com_err; + } + if (recover == 1 || result || + find_record_with_key(curr_file_info,buff)) + { + com_count[command][2]++; /* Mark error */ + break; + } + } + mi_result=mi_update(curr_file_info->isam,curr_file_info->record, + buff); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (mi_result) + com_count[command][2]++; /* Mark error */ + } + } + else + { + mi_result=mi_write(curr_file_info->isam,buff); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (mi_result) + com_count[command][2]++; /* Mark error */ + } + if (! recover && filepos != curr_file_info->isam->lastpos) + { + printf("Warning: Wrote at position: %s, should have been %s", + llstr(curr_file_info->isam->lastpos,llbuff), + llstr(filepos,llbuff2)); + goto com_err; + } + } + } + my_free(buff,MYF(0)); + break; + case MI_LOG_LOCK: + if (my_b_read(&cache,(byte*) head,sizeof(lock_command))) + goto err; + memcpy_fixed(&lock_command,head,sizeof(lock_command)); + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s(%d) -> %d\n",FILENAME(curr_file_info), + command_name[command],lock_command,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_lock_database(curr_file_info->isam,lock_command) != + (int) result) + goto com_err; + } + break; + case MI_LOG_DELETE_ALL: + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s -> %d\n",FILENAME(curr_file_info), + command_name[command],result); + break; + default: + VOID(fprintf(stderr, + "Error: found unknown command %d in logfile, aborted\n", + command)); + goto end; + } + } + end_key_cache(); + delete_tree(&tree); + VOID(end_io_cache(&cache)); + VOID(my_close(file,MYF(0))); + if (write_file && my_fclose(write_file,MYF(MY_WME))) + DBUG_RETURN(1); + DBUG_RETURN(0); + + err: + VOID(fprintf(stderr,"Got error %d when reading from logfile\n",my_errno)); + goto end; + com_err: + VOID(fprintf(stderr,"Got error %d, expected %d on command %s at %s\n", + my_errno,result,command_name[command], + llstr(isamlog_filepos,llbuff))); + end: + end_key_cache(); + delete_tree(&tree); + VOID(end_io_cache(&cache)); + VOID(my_close(file,MYF(0))); + if (write_file) + VOID(my_fclose(write_file,MYF(MY_WME))); + DBUG_RETURN(1); +} + + +static int read_string(IO_CACHE *file, register gptr *to, register uint length) +{ + DBUG_ENTER("read_string"); + + if (*to) + my_free((gptr) *to,MYF(0)); + if (!(*to= (gptr) my_malloc(length+1,MYF(MY_WME))) || + my_b_read(file,(byte*) *to,length)) + { + if (*to) + my_free(*to,MYF(0)); + *to= 0; + DBUG_RETURN(1); + } + *((char*) *to+length)= '\0'; + DBUG_RETURN (0); +} /* read_string */ + + +static int file_info_compare(void *a, void *b) +{ + long lint; + + if ((lint=((struct file_info*) a)->process - + ((struct file_info*) b)->process)) + return lint < 0L ? -1 : 1; + return ((struct file_info*) a)->filenr - ((struct file_info*) b)->filenr; +} + + /* ARGSUSED */ + +static int test_if_open (struct file_info *key, + element_count count __attribute__((unused)), + struct test_if_open_param *param) +{ + if (!strcmp(key->name,param->name) && key->id > param->max_id) + param->max_id=key->id; + return 0; +} + + +static void fix_blob_pointers(MI_INFO *info, byte *record) +{ + byte *pos; + MI_BLOB *blob,*end; + + pos=record+info->s->base.reclength; + for (end=info->blobs+info->s->base.blobs, blob= info->blobs; + blob != end ; + blob++) + { + memcpy_fixed(record+blob->offset+blob->pack_length,&pos,sizeof(char*)); + pos+=_mi_calc_blob_length(blob->pack_length,record+blob->offset); + } +} + +static uint set_maximum_open_files(uint maximum_files) +{ +#if defined(HAVE_GETRUSAGE) && defined(RLIMIT_NOFILE) + struct rlimit rlimit; + int old_max; + + if (maximum_files > MY_NFILE) + maximum_files=MY_NFILE; /* Don't crash my_open */ + + if (!getrlimit(RLIMIT_NOFILE,&rlimit)) + { + old_max=rlimit.rlim_max; + if (maximum_files && (int) maximum_files > old_max) + rlimit.rlim_max=maximum_files; + rlimit.rlim_cur=rlimit.rlim_max; + if (setrlimit(RLIMIT_NOFILE,&rlimit)) + { + if (old_max != (int) maximum_files) + { /* Set as much as we can */ + rlimit.rlim_max=rlimit.rlim_cur=old_max; + setrlimit(RLIMIT_NOFILE,&rlimit); + } + } + getrlimit(RLIMIT_NOFILE,&rlimit); /* Read if broken setrlimit */ + if (maximum_files && maximum_files < rlimit.rlim_cur) + VOID(fprintf(stderr,"Warning: Error from setrlimit: Max open files is %d\n",old_max)); + return rlimit.rlim_cur; + } +#endif + return min(maximum_files,MY_NFILE); +} + + /* close the file with hasn't been accessed for the longest time */ + /* ARGSUSED */ + +static int test_when_accessed (struct file_info *key, + element_count count __attribute__((unused)), + struct st_access_param *access_param) +{ + if (key->accessed < access_param->min_accessed && ! key->closed) + { + access_param->min_accessed=key->accessed; + access_param->found=key; + } + return 0; +} + + +static void file_info_free(struct file_info *fileinfo) +{ + DBUG_ENTER("file_info_free"); + if (update) + { + if (!fileinfo->closed) + VOID(mi_close(fileinfo->isam)); + if (fileinfo->record) + my_free(fileinfo->record,MYF(0)); + } + my_free(fileinfo->name,MYF(0)); + my_free(fileinfo->show_name,MYF(0)); + DBUG_VOID_RETURN; +} + + + +static int close_some_file(TREE *tree) +{ + struct st_access_param access_param; + + access_param.min_accessed=LONG_MAX; + access_param.found=0; + + VOID(tree_walk(tree,(tree_walk_action) test_when_accessed, + (void*) &access_param,left_root_right)); + if (!access_param.found) + return 1; /* No open file that is possibly to close */ + if (mi_close(access_param.found->isam)) + return 1; + access_param.found->closed=1; + return 0; +} + + +static int reopen_closed_file(TREE *tree, struct file_info *fileinfo) +{ + char name[FN_REFLEN]; + if (close_some_file(tree)) + return 1; /* No file to close */ + strmov(name,fileinfo->show_name); + if (fileinfo->id > 1) + *strrchr(name,'<')='\0'; /* Remove "<id>" */ + + if (!(fileinfo->isam= mi_open(name,O_RDWR,HA_OPEN_WAIT_IF_LOCKED))) + return 1; + fileinfo->closed=0; + re_open_count++; + return 0; +} + + /* Try to find record with uniq key */ + +static int find_record_with_key(struct file_info *file_info, byte *record) +{ + uint key; + MI_INFO *info=file_info->isam; + uchar tmp_key[MI_MAX_KEY_BUFF]; + + for (key=0 ; key < info->s->base.keys ; key++) + { + if ((((ulonglong) 1 << key) & info->s->state.key_map) && + info->s->keyinfo[key].flag & HA_NOSAME) + { + VOID(_mi_make_key(info,key,tmp_key,record,0L)); + return mi_rkey(info,file_info->record,(int) key,(char*) tmp_key,0, + HA_READ_KEY_EXACT); + } + } + return 1; +} + + +static void printf_log(const char *format,...) +{ + char llbuff[21]; + va_list args; + va_start(args,format); + if (verbose > 2) + printf("%9s:",llstr(isamlog_filepos,llbuff)); + if (verbose > 1) + printf("%5ld ",isamlog_process); /* Write process number */ + (void) vprintf((char*) format,args); + putchar('\n'); + va_end(args); +} + + +static bool cmp_filename(struct file_info *file_info, my_string name) +{ + if (!file_info) + return 1; + return strcmp(file_info->name,name) ? 1 : 0; +} diff --git a/myisam/myisampack.c b/myisam/myisampack.c new file mode 100644 index 00000000000..6f634f32ec9 --- /dev/null +++ b/myisam/myisampack.c @@ -0,0 +1,2140 @@ +/* Copyright (C) 1999 Monty Program KB + + This software is distributed with NO WARRANTY OF ANY KIND. No author or + distributor accepts any responsibility for the consequences of using it, or + for whether it serves any particular purpose or works at all, unless he or + she says so in writing. Refer to the Free Public License (the "License") + for full details. + Every copy of this file must include a copy of the License, normally in a + plain ASCII text file named PUBLIC. The License grants you the right to + copy, modify and redistribute this file, but only under certain conditions + described in the License. Among other things, the License requires that + the copyright notice and this notice be preserved on all copies. */ + +/* Pack isam file*/ + +#ifndef USE_MY_FUNC +#define USE_MY_FUNC /* We nead at least my_malloc */ +#endif + +#include "myisamdef.h" +#include <queues.h> +#include <my_tree.h> +#include "mysys_err.h" +#ifdef MSDOS +#include <io.h> +#endif +#ifndef __GNU_LIBRARY__ +#define __GNU_LIBRARY__ /* Skipp warnings in getopt.h */ +#endif +#include <getopt.h> + +#if INT_MAX > 32767 +#define BITS_SAVED 32 +#else +#define BITS_SAVED 16 +#endif + +#define IS_OFFSET ((uint) 32768) /* Bit if offset or char in tree */ +#define HEAD_LENGTH 32 +#define ALLOWED_JOIN_DIFF 256 /* Diff allowed to join trees */ + +#define DATA_TMP_EXT ".TMD" +#define OLD_EXT ".OLD" +#define WRITE_COUNT MY_HOW_OFTEN_TO_WRITE + +struct st_file_buffer { + File file; + char *buffer,*pos,*end; + my_off_t pos_in_file; + int bits; + uint byte; +}; + +struct st_huff_tree; +struct st_huff_element; + +typedef struct st_huff_counts { + uint field_length,max_zero_fill; + uint pack_type; + uint max_end_space,max_pre_space,length_bits,min_space; + ulong max_length; + enum en_fieldtype field_type; + struct st_huff_tree *tree; /* Tree for field */ + my_off_t counts[256]; + my_off_t end_space[8]; + my_off_t pre_space[8]; + my_off_t tot_end_space,tot_pre_space,zero_fields,empty_fields,bytes_packed; + TREE int_tree; + byte *tree_buff; + byte *tree_pos; +} HUFF_COUNTS; + +typedef struct st_huff_element HUFF_ELEMENT; + +struct st_huff_element { + my_off_t count; + union un_element { + struct st_nod { + HUFF_ELEMENT *left,*right; + } nod; + struct st_leaf { + HUFF_ELEMENT *null; + uint element_nr; /* Number of element */ + } leaf; + } a; +}; + + +typedef struct st_huff_tree { + HUFF_ELEMENT *root,*element_buffer; + HUFF_COUNTS *counts; + uint tree_number; + uint elements; + my_off_t bytes_packed; + uint tree_pack_length; + uint min_chr,max_chr,char_bits,offset_bits,max_offset,height; + ulong *code; + uchar *code_len; +} HUFF_TREE; + + +typedef struct st_isam_mrg { + MI_INFO **file,**current,**end; + uint free_file; + uint count; + uint min_pack_length; /* Theese is used by packed data */ + uint max_pack_length; + uint ref_length; + uint max_blob_length; + my_off_t records; +} MRG_INFO; + + +extern int main(int argc,char * *argv); +static void get_options(int *argc,char ***argv); +static MI_INFO *open_isam_file(char *name,int mode); +static bool open_isam_files(MRG_INFO *mrg,char **names,uint count); +static int compress(MRG_INFO *file,char *join_name); +static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records); +static void free_counts_and_tree_and_queue(HUFF_TREE *huff_trees, + uint trees, + HUFF_COUNTS *huff_counts, + uint fields); +static int compare_tree(const uchar *s,const uchar *t); +static int get_statistic(MRG_INFO *mrg,HUFF_COUNTS *huff_counts); +static void check_counts(HUFF_COUNTS *huff_counts,uint trees, + my_off_t records); +static int test_space_compress(HUFF_COUNTS *huff_counts,my_off_t records, + uint max_space_length,my_off_t *space_counts, + my_off_t tot_space_count, + enum en_fieldtype field_type); +static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts,uint trees); +static int make_huff_tree(HUFF_TREE *tree,HUFF_COUNTS *huff_counts); +static int compare_huff_elements(void *not_used, byte *a,byte *b); +static int save_counts_in_queue(byte *key,element_count count, + HUFF_TREE *tree); +static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts,uint flag); +static uint join_same_trees(HUFF_COUNTS *huff_counts,uint trees); +static int make_huff_decode_table(HUFF_TREE *huff_tree,uint trees); +static void make_traverse_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element,uint size, + ulong code); +static int write_header(MRG_INFO *isam_file, uint header_length,uint trees, + my_off_t tot_elements,my_off_t filelength); +static void write_field_info(HUFF_COUNTS *counts, uint fields,uint trees); +static my_off_t write_huff_tree(HUFF_TREE *huff_tree,uint trees); +static uint *make_offset_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element, + uint *offset); +static uint max_bit(uint value); +static int compress_isam_file(MRG_INFO *file,HUFF_COUNTS *huff_counts); +static char *make_new_name(char *new_name,char *old_name); +static char *make_old_name(char *new_name,char *old_name); +static void init_file_buffer(File file,pbool read_buffer); +static int flush_buffer(ulong neaded_length); +static void end_file_buffer(void); +static void write_bits(ulong value,uint bits); +static void flush_bits(void); +static int save_state(MI_INFO *isam_file,MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc); +static int save_state_mrg(File file,MRG_INFO *isam_file,my_off_t new_length, + ha_checksum crc); +static int mrg_close(MRG_INFO *mrg); +static int mrg_rrnd(MRG_INFO *info,byte *buf); +static void mrg_reset(MRG_INFO *mrg); + + +static int backup=0,error_on_write=0,test_only=0,verbose=0,silent=0, + write_loop=0,force_pack=0,opt_wait=0,isamchk_neaded=0; +static int tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; +static uint tree_buff_length=8196-MALLOC_OVERHEAD; +static char tmp_dir[FN_REFLEN]={0},*join_table; +static my_off_t intervall_length; +static ha_checksum glob_crc; +static struct st_file_buffer file_buffer; +static QUEUE queue; +static HUFF_COUNTS *global_count; +static char zero_string[]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +static const char *load_default_groups[]= { "myisampack",0 }; + + /* The main program */ + +int main(int argc, char **argv) +{ + int error,ok; + MRG_INFO merge; + char **default_argv; + MY_INIT(argv[0]); + + load_defaults("my",load_default_groups,&argc,&argv); + default_argv= argv; + get_options(&argc,&argv); + + error=ok=isamchk_neaded=0; + if (join_table) + { /* Join files into one */ + if (open_isam_files(&merge,argv,(uint) argc) || + compress(&merge,join_table)) + error=1; + } + else while (argc--) + { + MI_INFO *isam_file; + if (!(isam_file=open_isam_file(*argv++,O_RDWR))) + error=1; + else + { + merge.file= &isam_file; + merge.current=0; + merge.free_file=0; + merge.count=1; + if (compress(&merge,0)) + error=1; + else + ok=1; + } + } + if (ok && isamchk_neaded && !silent) + puts("Remember to run myisamchk -rq on compressed tables"); + VOID(fflush(stdout)); VOID(fflush(stderr)); + free_defaults(default_argv); + my_end(verbose ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + exit(error ? 2 : 0); +#ifndef _lint + return 0; /* No compiler warning */ +#endif +} + + +static struct option long_options[] = +{ + {"backup", no_argument, 0, 'b'}, + {"debug", optional_argument, 0, '#'}, + {"force", no_argument, 0, 'f'}, + {"join", required_argument, 0, 'j'}, + {"help", no_argument, 0, '?'}, + {"packlength",required_argument, 0, 'p'}, + {"silent", no_argument, 0, 's'}, + {"tmpdir", required_argument, 0, 'T'}, + {"test", no_argument, 0, 't'}, + {"verbose", no_argument, 0, 'v'}, + {"version", no_argument, 0, 'V'}, + {"wait", no_argument, 0, 'w'}, + {0, 0, 0, 0} +}; + +static void print_version(void) +{ + printf("%s Ver 1.7 for %s on %s\n",my_progname,SYSTEM_TYPE,MACHINE_TYPE); +} + +static void usage(void) +{ + print_version(); + puts("Copyright (C) 1999-2000 Monty Program KB."); + puts("This is not free software. You must have a licence to use this program"); + puts("This software comes with ABSOLUTELY NO WARRANTY\n"); + puts("Pack a MyISAM-table to take much smaller space"); + puts("Keys are not updated, one must run myisamchk -rq on datafile afterwards"); + puts("You should give the .MSI file as the filename argument"); + + printf("\nUsage: %s [OPTIONS] filename...\n", my_progname); + puts("\n\ + -b, --backup Make a backup of the table as table_name.OLD\n\ + -f, --force Force packing of table even if it's gets bigger or\n\ + tempfile exists.\n\ + -j, --join='new_table_name'\n\ + Join all given tables into 'new_table_name'.\n\ + All tables MUST have the identical layout.\n\ + -s, --silent Be more silent.\n\ + -t, --test Don't pack table, only test packing it\n\ + -v, --verbose Write info about progress and packing result\n\ + -w, --wait Wait and retry if table is in use\n\ + -T, --tmpdir=# Use temporary directory to store temporary table\n\ + -#, --debug=... output debug log. Often this is 'd:t:o,filename`\n\ + -?, --help display this help and exit\n\ + -V, --version output version information and exit"); + print_defaults("my",load_default_groups); +}; + + /* reads options */ + /* Initiates DEBUG - but no debugging here ! */ + +static void get_options(int *argc,char ***argv) +{ + int c,option_index=0; + uint length; + + my_progname= argv[0][0]; + if (isatty(fileno(stdout))) + write_loop=1; + + while ((c=getopt_long(*argc,*argv,"bfj:stvwT:#::?V",long_options, + &option_index)) != EOF) + { + switch(c) { + case 'b': + backup=1; + break; + case 'f': + force_pack=1; + tmpfile_createflag=O_RDWR | O_TRUNC; + break; + case 'j': + join_table=optarg; + break; + case 's': + write_loop=verbose=0; silent=1; + break; + case 't': + test_only=verbose=1; + break; + case 'T': + length=(uint) (strmov(tmp_dir,optarg)-tmp_dir); + if (length != dirname_length(tmp_dir)) + { + tmp_dir[length]=FN_LIBCHAR; + tmp_dir[length+1]=0; + } + break; + case 'v': + verbose=1; silent=0; + break; + case 'w': + opt_wait=1; + break; + case '#': + DBUG_PUSH(optarg ? optarg : "d:t:o"); + break; + case 'V': print_version(); exit(0); + case 'I': + case '?': + usage(); + exit(0); + default: + fprintf(stderr,"%s: Illegal option: -%c\n",my_progname,opterr); + usage(); + exit(1); + } + } + (*argc)-=optind; + (*argv)+=optind; + if (!*argc) + { + usage(); + exit(1); + } + if (join_table) + { + backup=0; /* Not needed */ + tmp_dir[0]=0; + } + return; +} + + +static MI_INFO *open_isam_file(char *name,int mode) +{ + MI_INFO *isam_file; + MYISAM_SHARE *share; + DBUG_ENTER("open_isam_file"); + + if (!(isam_file=mi_open(name,mode, + (opt_wait ? HA_OPEN_WAIT_IF_LOCKED : + HA_OPEN_ABORT_IF_LOCKED)))) + { + VOID(fprintf(stderr,"%s gave error %d on open\n",name,my_errno)); + DBUG_RETURN(0); + } + share=isam_file->s; + if (share->options & HA_OPTION_COMPRESS_RECORD && !join_table) + { + if (!force_pack) + { + VOID(fprintf(stderr,"%s is already compressed\n",name)); + VOID(mi_close(isam_file)); + DBUG_RETURN(0); + } + if (verbose) + puts("Recompressing already compressed table"); + share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */ + } + if (! force_pack && share->state.state.records != 0 && + (share->state.state.records <= 1 || + share->state.state.data_file_length < 1024)) + { + VOID(fprintf(stderr,"%s is too small to compress\n",name)); + VOID(mi_close(isam_file)); + DBUG_RETURN(0); + } + VOID(mi_lock_database(isam_file,F_WRLCK)); + DBUG_RETURN(isam_file); +} + + +static bool open_isam_files(MRG_INFO *mrg,char **names,uint count) +{ + uint i,j; + mrg->count=0; + mrg->current=0; + mrg->file=(MI_INFO**) my_malloc(sizeof(MI_INFO*)*count,MYF(MY_FAE)); + mrg->free_file=1; + for (i=0; i < count ; i++) + { + if (!(mrg->file[i]=open_isam_file(names[i],O_RDONLY))) + goto error; + } + /* Check that files are identical */ + for (j=0 ; j < count-1 ; j++) + { + MI_COLUMNDEF *m1,*m2,*end; + if (mrg->file[j]->s->base.reclength != mrg->file[j+1]->s->base.reclength || + mrg->file[j]->s->base.fields != mrg->file[j+1]->s->base.fields) + goto diff_file; + m1=mrg->file[j]->s->rec; + end=m1+mrg->file[j]->s->base.fields; + m2=mrg->file[j+1]->s->rec; + for ( ; m1 != end ; m1++,m2++) + { + if (m1->type != m2->type || m1->length != m2->length) + goto diff_file; + } + } + mrg->count=count; + return 0; + + diff_file: + fprintf(stderr,"%s: Tables '%s' and '%s' are not identical\n", + my_progname,names[j],names[j+1]); + error: + while (i--) + mi_close(mrg->file[i]); + my_free((gptr) mrg->file,MYF(0)); + return 1; +} + + +static int compress(MRG_INFO *mrg,char *result_table) +{ + int error; + File new_file,join_isam_file; + MI_INFO *isam_file; + MYISAM_SHARE *share; + char org_name[FN_REFLEN],new_name[FN_REFLEN],temp_name[FN_REFLEN]; + uint i,header_length,fields,trees,used_trees; + my_off_t old_length,new_length,tot_elements; + HUFF_COUNTS *huff_counts; + HUFF_TREE *huff_trees; + DBUG_ENTER("compress"); + + isam_file=mrg->file[0]; /* Take this as an example */ + share=isam_file->s; + new_file=join_isam_file= -1; + trees=fields=0; + huff_trees=0; + huff_counts=0; + + /* Create temporary or join file */ + + if (backup) + VOID(fn_format(org_name,isam_file->filename,"",MI_NAME_DEXT,2)); + else + VOID(fn_format(org_name,isam_file->filename,"",MI_NAME_DEXT,2+4+16)); + if (!test_only && result_table) + { + /* Make a new indexfile based on first file in list */ + uint length; + char *buff; + strmov(org_name,result_table); /* Fix error messages */ + VOID(fn_format(new_name,result_table,"",MI_NAME_IEXT,2)); + if ((join_isam_file=my_create(new_name,0,tmpfile_createflag,MYF(MY_WME))) + < 0) + goto err; + length=(uint) share->base.keystart; + if (!(buff=my_malloc(length,MYF(MY_WME)))) + goto err; + if (my_pread(share->kfile,buff,length,0L,MYF(MY_WME | MY_NABP)) || + my_write(join_isam_file,buff,length, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL))) + { + my_free(buff,MYF(0)); + goto err; + } + my_free(buff,MYF(0)); + VOID(fn_format(new_name,result_table,"",MI_NAME_DEXT,2)); + } + else if (!tmp_dir[0]) + VOID(make_new_name(new_name,org_name)); + else + VOID(fn_format(new_name,org_name,tmp_dir,DATA_TMP_EXT,1+2+4)); + if (!test_only && + (new_file=my_create(new_name,0,tmpfile_createflag,MYF(MY_WME))) < 0) + goto err; + + /* Start calculating statistics */ + + mrg->records=0; + for (i=0 ; i < mrg->count ; i++) + mrg->records+=mrg->file[i]->s->state.state.records; + if (write_loop || verbose) + { + printf("Compressing %s: (%lu records)\n", + result_table ? new_name : org_name,(ulong) mrg->records); + } + trees=fields=share->base.fields; + huff_counts=init_huff_count(isam_file,mrg->records); + QUICK_SAFEMALLOC; + if (write_loop || verbose) + printf("- Calculating statistics\n"); + if (get_statistic(mrg,huff_counts)) + goto err; + NORMAL_SAFEMALLOC; + old_length=0; + for (i=0; i < mrg->count ; i++) + old_length+= (mrg->file[i]->s->state.state.data_file_length - + mrg->file[i]->s->state.state.empty); + + if (init_queue(&queue,256,0,0,compare_huff_elements,0)) + goto err; + check_counts(huff_counts,fields,mrg->records); + huff_trees=make_huff_trees(huff_counts,trees); + if ((int) (used_trees=join_same_trees(huff_counts,trees)) < 0) + goto err; + if (make_huff_decode_table(huff_trees,fields)) + goto err; + + init_file_buffer(new_file,0); + file_buffer.pos_in_file=HEAD_LENGTH; + if (! test_only) + VOID(my_seek(new_file,file_buffer.pos_in_file,MY_SEEK_SET,MYF(0))); + + write_field_info(huff_counts,fields,used_trees); + if (!(tot_elements=write_huff_tree(huff_trees,trees))) + goto err; + header_length=(uint) file_buffer.pos_in_file+ + (uint) (file_buffer.pos-file_buffer.buffer); + + /* Compress file */ + if (write_loop || verbose) + printf("- Compressing file\n"); + error=compress_isam_file(mrg,huff_counts); + new_length=file_buffer.pos_in_file; + if (!error && !test_only) + { + char buff[MEMMAP_EXTRA_MARGIN]; /* End marginal for memmap */ + bzero(buff,sizeof(buff)); + error=my_write(file_buffer.file,buff,sizeof(buff), + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; + } + if (!error) + error=write_header(mrg,header_length,used_trees,tot_elements, + new_length); + end_file_buffer(); + + if (verbose && mrg->records) + printf("Min record length: %6d Max length: %6d Mean total length: %6ld\n", + mrg->min_pack_length,mrg->max_pack_length, + (ulong) (new_length/mrg->records)); + + if (!test_only) + { + error|=my_close(new_file,MYF(MY_WME)); + if (!result_table) + { + error|=my_close(isam_file->dfile,MYF(MY_WME)); + isam_file->dfile= -1; /* Tell mi_close file is closed */ + } + } + + free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); + if (! test_only && ! error) + { + if (result_table) + { + error=save_state_mrg(join_isam_file,mrg,new_length,glob_crc); + } + else + { + if (backup) + { + if (my_rename(org_name,make_old_name(temp_name,isam_file->filename), + MYF(MY_WME))) + error=1; + else + { + if (tmp_dir[0]) + { + if (!(error=my_copy(new_name,org_name,MYF(MY_WME)))) + VOID(my_delete(new_name,MYF(MY_WME))); + } + else + error=my_rename(new_name,org_name,MYF(MY_WME)); + if (!error) + VOID(my_copystat(temp_name,org_name,MYF(MY_COPYTIME))); + } + } + else + { + if (tmp_dir[0]) + { + + if (!(error=my_copy(new_name,org_name, + MYF(MY_WME | MY_HOLD_ORIGINAL_MODES + | MY_COPYTIME)))) + VOID(my_delete(new_name,MYF(MY_WME))); + } + else + error=my_redel(org_name,new_name,MYF(MY_WME | MY_COPYTIME)); + } + if (! error) + error=save_state(isam_file,mrg,new_length,glob_crc); + } + } + error|=mrg_close(mrg); + if (join_isam_file >= 0) + error|=my_close(join_isam_file,MYF(MY_WME)); + if (error) + { + VOID(fprintf(stderr,"Aborting: %s is not compressed\n",org_name)); + DBUG_RETURN(-1); + } + if (write_loop || verbose) + { + if (old_length) + printf("%.4g%% \n", (((longlong) (old_length -new_length))*100.0/ + (longlong) old_length)); + else + puts("Empty file saved in compressed format"); + } + DBUG_RETURN(0); + + err: + free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); + if (new_file >= 0) + VOID(my_close(new_file,MYF(0))); + if (join_isam_file >= 0) + VOID(my_close(join_isam_file,MYF(0))); + mrg_close(mrg); + VOID(fprintf(stderr,"Aborted: %s is not compressed\n",org_name)); + DBUG_RETURN(-1); +} + + /* Init a huff_count-struct for each field and init it */ + +static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records) +{ + reg2 uint i; + reg1 HUFF_COUNTS *count; + if ((count = (HUFF_COUNTS*) my_malloc(info->s->base.fields* + sizeof(HUFF_COUNTS), + MYF(MY_ZEROFILL | MY_WME)))) + { + for (i=0 ; i < info->s->base.fields ; i++) + { + enum en_fieldtype type; + count[i].field_length=info->s->rec[i].length; + type= count[i].field_type= (enum en_fieldtype) info->s->rec[i].type; + if (type == FIELD_INTERVALL || + type == FIELD_CONSTANT || + type == FIELD_ZERO) + type = FIELD_NORMAL; + if (count[i].field_length <= 8 && + (type == FIELD_NORMAL || + type == FIELD_SKIPP_ZERO)) + count[i].max_zero_fill= count[i].field_length; + init_tree(&count[i].int_tree,0,-1,(qsort_cmp) compare_tree,0,NULL); + if (records && type != FIELD_BLOB && type != FIELD_VARCHAR) + count[i].tree_pos=count[i].tree_buff = + my_malloc(count[i].field_length > 1 ? tree_buff_length : 2, + MYF(MY_WME)); + } + } + return count; +} + + + /* Free memory used by counts and trees */ + +static void free_counts_and_tree_and_queue(HUFF_TREE *huff_trees, uint trees, + HUFF_COUNTS *huff_counts, + uint fields) +{ + register uint i; + + if (huff_trees) + { + for (i=0 ; i < trees ; i++) + { + if (huff_trees[i].element_buffer) + my_free((gptr) huff_trees[i].element_buffer,MYF(0)); + if (huff_trees[i].code) + my_free((gptr) huff_trees[i].code,MYF(0)); + } + my_free((gptr) huff_trees,MYF(0)); + } + if (huff_counts) + { + for (i=0 ; i < fields ; i++) + { + if (huff_counts[i].tree_buff) + { + my_free((gptr) huff_counts[i].tree_buff,MYF(0)); + delete_tree(&huff_counts[i].int_tree); + } + } + my_free((gptr) huff_counts,MYF(0)); + } + delete_queue(&queue); /* This is safe to free */ + return; +} + + /* Read through old file and gather some statistics */ + +static int get_statistic(MRG_INFO *mrg,HUFF_COUNTS *huff_counts) +{ + int error; + uint length; + ulong reclength,max_blob_length; + byte *record,*pos,*next_pos,*end_pos,*start_pos; + ha_rows record_count; + my_bool static_row_size; + HUFF_COUNTS *count,*end_count; + TREE_ELEMENT *element; + DBUG_ENTER("get_statistic"); + + reclength=mrg->file[0]->s->base.reclength; + record=(byte*) my_alloca(reclength); + end_count=huff_counts+mrg->file[0]->s->base.fields; + record_count=0; glob_crc=0; + max_blob_length=0; + + /* Check how to calculate checksum */ + static_row_size=1; + for (count=huff_counts ; count < end_count ; count++) + { + if (count->field_type == FIELD_BLOB || count->field_type == FIELD_VARCHAR) + { + static_row_size=0; + break; + } + } + + mrg_reset(mrg); + while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) + { + ulong tot_blob_length=0; + if (! error) + { + if (static_row_size) + glob_crc+=mi_static_checksum(mrg->file[0],record); + else + glob_crc+=mi_checksum(mrg->file[0],record); + for (pos=record,count=huff_counts ; + count < end_count ; + count++, + pos=next_pos) + { + next_pos=end_pos=(start_pos=pos)+count->field_length; + + /* Put value in tree if there is room for it */ + if (count->tree_buff) + { + global_count=count; + if (!(element=tree_insert(&count->int_tree,pos,0)) || + (element->count == 1 && + count->tree_buff + tree_buff_length < + count->tree_pos + count->field_length) || + (count->field_length == 1 && + count->int_tree.elements_in_tree > 1)) + { + delete_tree(&count->int_tree); + my_free(count->tree_buff,MYF(0)); + count->tree_buff=0; + } + else + { + if (element->count == 1) + { /* New element */ + memcpy(count->tree_pos,pos,(size_t) count->field_length); + tree_set_pointer(element,count->tree_pos); + count->tree_pos+=count->field_length; + } + } + } + + /* Save character counters and space-counts and zero-field-counts */ + if (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIPP_ENDSPACE) + { + for ( ; end_pos > pos ; end_pos--) + if (end_pos[-1] != ' ') + break; + if (end_pos == pos) + { + count->empty_fields++; + count->max_zero_fill=0; + continue; + } + length= (uint) (next_pos-end_pos); + count->tot_end_space+=length; + if (length < 8) + count->end_space[length]++; + if (count->max_end_space < length) + count->max_end_space = length; + } + if (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIPP_PRESPACE) + { + for (pos=start_pos; pos < end_pos ; pos++) + if (pos[0] != ' ') + break; + if (end_pos == pos) + { + count->empty_fields++; + count->max_zero_fill=0; + continue; + } + length= (uint) (pos-start_pos); + count->tot_pre_space+=length; + if (length < 8) + count->pre_space[length]++; + if (count->max_pre_space < length) + count->max_pre_space = length; + } + if (count->field_type == FIELD_BLOB) + { + uint field_length=count->field_length -mi_portable_sizeof_char_ptr; + ulong blob_length= _mi_calc_blob_length(field_length, start_pos); + memcpy_fixed((char*) &pos, start_pos+field_length,sizeof(char*)); + end_pos=pos+blob_length; + tot_blob_length+=blob_length; + set_if_bigger(count->max_length,blob_length); + } + else if (count->field_type == FIELD_VARCHAR) + { + length=uint2korr(start_pos); + pos=start_pos+2; + end_pos=start_pos+length; + set_if_bigger(count->max_length,length); + } + if (count->field_length <= 8 && + (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIPP_ZERO)) + { + uint i; + if (!memcmp((byte*) start_pos,zero_string,count->field_length)) + { + count->zero_fields++; + continue; + } + for (i =0 ; i < count->max_zero_fill && ! end_pos[-1 - (int) i] ; + i++) ; + if (i < count->max_zero_fill) + count->max_zero_fill=i; + } + if (count->field_type == FIELD_ZERO || + count->field_type == FIELD_CHECK) + continue; + for ( ; pos < end_pos ; pos++) + count->counts[(uchar) *pos]++; + } + if (tot_blob_length > max_blob_length) + max_blob_length=tot_blob_length; + record_count++; + if (write_loop && record_count % WRITE_COUNT == 0) + { + printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + } + } + else if (error != HA_ERR_RECORD_DELETED) + { + fprintf(stderr,"Got error %d while reading rows",error); + break; + } + } + if (write_loop) + { + printf(" \r"); VOID(fflush(stdout)); + } + mrg->records=record_count; + mrg->max_blob_length=max_blob_length; + my_afree((gptr) record); + DBUG_RETURN(error != HA_ERR_END_OF_FILE); +} + +static int compare_huff_elements(void *not_used, byte *a, byte *b) +{ + return *((my_off_t*) a) < *((my_off_t*) b) ? -1 : + (*((my_off_t*) a) == *((my_off_t*) b) ? 0 : 1); +} + + /* Check each tree if we should use pre-space-compress, end-space- + compress, empty-field-compress or zero-field-compress */ + +static void check_counts(HUFF_COUNTS *huff_counts, uint trees, + my_off_t records) +{ + uint space_fields,fill_zero_fields,field_count[(int) FIELD_VARCHAR+1]; + my_off_t old_length,new_length,length; + DBUG_ENTER("check_counts"); + + bzero((gptr) field_count,sizeof(field_count)); + space_fields=fill_zero_fields=0; + + for (; trees-- ; huff_counts++) + { + if (huff_counts->field_type == FIELD_BLOB) + { + huff_counts->length_bits=max_bit(huff_counts->max_length); + goto found_pack; + } + else if (huff_counts->field_type == FIELD_VARCHAR) + { + huff_counts->length_bits=max_bit(huff_counts->max_length); + goto found_pack; + } + else if (huff_counts->field_type == FIELD_CHECK) + { + huff_counts->bytes_packed=0; + huff_counts->counts[0]=0; + goto found_pack; + } + + huff_counts->field_type=FIELD_NORMAL; + huff_counts->pack_type=0; + + if (huff_counts->zero_fields || ! records) + { + my_off_t old_space_count; + if (huff_counts->zero_fields == records) + { + huff_counts->field_type= FIELD_ZERO; + huff_counts->bytes_packed=0; + huff_counts->counts[0]=0; + goto found_pack; + } + old_space_count=huff_counts->counts[' ']; + huff_counts->counts[' ']+=huff_counts->tot_end_space+ + huff_counts->tot_pre_space + + huff_counts->empty_fields * huff_counts->field_length; + old_length=calc_packed_length(huff_counts,0)+records/8; + length=huff_counts->zero_fields*huff_counts->field_length; + huff_counts->counts[0]+=length; + new_length=calc_packed_length(huff_counts,0); + if (old_length < new_length && huff_counts->field_length > 1) + { + huff_counts->field_type=FIELD_SKIPP_ZERO; + huff_counts->counts[0]-=length; + huff_counts->bytes_packed=old_length- records/8; + goto found_pack; + } + huff_counts->counts[' ']=old_space_count; + } + huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + if (huff_counts->empty_fields) + { + if (huff_counts->field_length > 2 && + huff_counts->empty_fields + (records - huff_counts->empty_fields)* + (1+max_bit(max(huff_counts->max_pre_space, + huff_counts->max_end_space))) < + records * max_bit(huff_counts->field_length)) + { + huff_counts->pack_type |= PACK_TYPE_SPACE_FIELDS; + } + else + { + length=huff_counts->empty_fields*huff_counts->field_length; + if (huff_counts->tot_end_space || ! huff_counts->tot_pre_space) + { + huff_counts->tot_end_space+=length; + huff_counts->max_end_space=huff_counts->field_length; + if (huff_counts->field_length < 8) + huff_counts->end_space[huff_counts->field_length]+= + huff_counts->empty_fields; + } + else + { + huff_counts->tot_pre_space+=length; + huff_counts->max_pre_space=huff_counts->field_length; + if (huff_counts->field_length < 8) + huff_counts->pre_space[huff_counts->field_length]+= + huff_counts->empty_fields; + } + } + } + if (huff_counts->tot_end_space) + { + huff_counts->counts[' ']+=huff_counts->tot_pre_space; + if (test_space_compress(huff_counts,records,huff_counts->max_end_space, + huff_counts->end_space, + huff_counts->tot_end_space,FIELD_SKIPP_ENDSPACE)) + goto found_pack; + huff_counts->counts[' ']-=huff_counts->tot_pre_space; + } + if (huff_counts->tot_pre_space) + { + if (test_space_compress(huff_counts,records,huff_counts->max_pre_space, + huff_counts->pre_space, + huff_counts->tot_pre_space,FIELD_SKIPP_PRESPACE)) + goto found_pack; + } + + found_pack: /* Found field-packing */ + + /* Test if we can use zero-fill */ + + if (huff_counts->max_zero_fill && + (huff_counts->field_type == FIELD_NORMAL || + huff_counts->field_type == FIELD_SKIPP_ZERO)) + { + huff_counts->counts[0]-=huff_counts->max_zero_fill* + (huff_counts->field_type == FIELD_SKIPP_ZERO ? + records - huff_counts->zero_fields : records); + huff_counts->pack_type|=PACK_TYPE_ZERO_FILL; + huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + } + + /* Test if intervall-field is better */ + + if (huff_counts->tree_buff) + { + HUFF_TREE tree; + + tree.element_buffer=0; + if (!make_huff_tree(&tree,huff_counts) && + tree.bytes_packed+tree.tree_pack_length < huff_counts->bytes_packed) + { + if (tree.elements == 1) + huff_counts->field_type=FIELD_CONSTANT; + else + huff_counts->field_type=FIELD_INTERVALL; + huff_counts->pack_type=0; + } + else + { + my_free((gptr) huff_counts->tree_buff,MYF(0)); + delete_tree(&huff_counts->int_tree); + huff_counts->tree_buff=0; + } + if (tree.element_buffer) + my_free((gptr) tree.element_buffer,MYF(0)); + } + if (huff_counts->pack_type & PACK_TYPE_SPACE_FIELDS) + space_fields++; + if (huff_counts->pack_type & PACK_TYPE_ZERO_FILL) + fill_zero_fields++; + field_count[huff_counts->field_type]++; + } + if (verbose) + printf("\nnormal: %3d empty-space: %3d empty-zero: %3d empty-fill: %3d\npre-space: %3d end-space: %3d intervall-fields: %3d zero: %3d\n", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIPP_ZERO],fill_zero_fields, + field_count[FIELD_SKIPP_PRESPACE], + field_count[FIELD_SKIPP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO]); + DBUG_VOID_RETURN; +} + + /* Test if we can use space-compression and empty-field-compression */ + +static int +test_space_compress(HUFF_COUNTS *huff_counts, my_off_t records, + uint max_space_length, my_off_t *space_counts, + my_off_t tot_space_count, enum en_fieldtype field_type) +{ + int min_pos; + uint length_bits,i; + my_off_t space_count,min_space_count,min_pack,new_length,skipp; + + length_bits=max_bit(max_space_length); + + /* Default no end_space-packing */ + space_count=huff_counts->counts[(uint) ' ']; + min_space_count= (huff_counts->counts[(uint) ' ']+= tot_space_count); + min_pack=calc_packed_length(huff_counts,0); + min_pos= -2; + huff_counts->counts[(uint) ' ']=space_count; + + /* Test with allways space-count */ + new_length=huff_counts->bytes_packed+length_bits*records/8; + if (new_length+1 < min_pack) + { + min_pos= -1; + min_pack=new_length; + min_space_count=space_count; + } + /* Test with length-flag */ + for (skipp=0L, i=0 ; i < 8 ; i++) + { + if (space_counts[i]) + { + if (i) + huff_counts->counts[(uint) ' ']+=space_counts[i]; + skipp+=huff_counts->pre_space[i]; + new_length=calc_packed_length(huff_counts,0)+ + (records+(records-skipp)*(1+length_bits))/8; + if (new_length < min_pack) + { + min_pos=(int) i; + min_pack=new_length; + min_space_count=huff_counts->counts[(uint) ' ']; + } + } + } + + huff_counts->counts[(uint) ' ']=min_space_count; + huff_counts->bytes_packed=min_pack; + switch (min_pos) { + case -2: + return(0); /* No space-compress */ + case -1: /* Always space-count */ + huff_counts->field_type=field_type; + huff_counts->min_space=0; + huff_counts->length_bits=max_bit(max_space_length); + break; + default: + huff_counts->field_type=field_type; + huff_counts->min_space=(uint) min_pos; + huff_counts->pack_type|=PACK_TYPE_SELECTED; + huff_counts->length_bits=max_bit(max_space_length); + break; + } + return(1); /* Using space-compress */ +} + + + /* Make a huff_tree of each huff_count */ + +static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts, uint trees) +{ + uint tree; + HUFF_TREE *huff_tree; + DBUG_ENTER("make_huff_trees"); + + if (!(huff_tree=(HUFF_TREE*) my_malloc(trees*sizeof(HUFF_TREE), + MYF(MY_WME | MY_ZEROFILL)))) + DBUG_RETURN(0); + + for (tree=0 ; tree < trees ; tree++) + { + if (make_huff_tree(huff_tree+tree,huff_counts+tree)) + { + while (tree--) + my_free((gptr) huff_tree[tree].element_buffer,MYF(0)); + my_free((gptr) huff_tree,MYF(0)); + DBUG_RETURN(0); + } + } + DBUG_RETURN(huff_tree); +} + + /* Update huff_tree according to huff_counts->counts or + huff_counts->tree_buff */ + +static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) +{ + uint i,found,bits_packed,first,last; + my_off_t bytes_packed; + HUFF_ELEMENT *a,*b,*new; + + first=last=0; + if (huff_counts->tree_buff) + { + found= (uint) (huff_counts->tree_pos - huff_counts->tree_buff) / + huff_counts->field_length; + first=0; last=found-1; + } + else + { + for (i=found=0 ; i < 256 ; i++) + { + if (huff_counts->counts[i]) + { + if (! found++) + first=i; + last=i; + } + } + if (found < 2) + found=2; + } + + if (queue.max_elements < found) + { + delete_queue(&queue); + if (init_queue(&queue,found,0,0,compare_huff_elements,0)) + return -1; + } + + if (!huff_tree->element_buffer) + { + if (!(huff_tree->element_buffer= + (HUFF_ELEMENT*) my_malloc(found*2*sizeof(HUFF_ELEMENT),MYF(MY_WME)))) + return 1; + } + else + { + HUFF_ELEMENT *temp; + if (!(temp= + (HUFF_ELEMENT*) my_realloc((gptr) huff_tree->element_buffer, + found*2*sizeof(HUFF_ELEMENT), + MYF(MY_WME)))) + return 1; + huff_tree->element_buffer=temp; + } + + huff_counts->tree=huff_tree; + huff_tree->counts=huff_counts; + huff_tree->min_chr=first; + huff_tree->max_chr=last; + huff_tree->char_bits=max_bit(last-first); + huff_tree->offset_bits=max_bit(found-1)+1; + + if (huff_counts->tree_buff) + { + huff_tree->elements=0; + tree_walk(&huff_counts->int_tree, + (int (*)(void*, element_count,void*)) save_counts_in_queue, + (gptr) huff_tree, left_root_right); + huff_tree->tree_pack_length=(1+15+16+5+5+ + (huff_tree->char_bits+1)*found+ + (huff_tree->offset_bits+1)* + (found-2)+7)/8 + + (uint) (huff_tree->counts->tree_pos- + huff_tree->counts->tree_buff); + } + else + { + huff_tree->elements=found; + huff_tree->tree_pack_length=(9+9+5+5+ + (huff_tree->char_bits+1)*found+ + (huff_tree->offset_bits+1)* + (found-2)+7)/8; + + for (i=first, found=0 ; i <= last ; i++) + { + if (huff_counts->counts[i]) + { + new=huff_tree->element_buffer+(found++); + new->count=huff_counts->counts[i]; + new->a.leaf.null=0; + new->a.leaf.element_nr=i; + queue.root[found]=(byte*) new; + } + } + while (found < 2) + { /* Our huff_trees request at least 2 elements */ + new=huff_tree->element_buffer+(found++); + new->count=0; + new->a.leaf.null=0; + if (last) + new->a.leaf.element_nr=huff_tree->min_chr=last-1; + else + new->a.leaf.element_nr=huff_tree->max_chr=last+1; + queue.root[found]=(byte*) new; + } + } + queue.elements=found; + + for (i=found/2 ; i > 0 ; i--) + _downheap(&queue,i); + bytes_packed=0; bits_packed=0; + for (i=1 ; i < found ; i++) + { + a=(HUFF_ELEMENT*) queue_remove(&queue,0); + b=(HUFF_ELEMENT*) queue.root[1]; + new=huff_tree->element_buffer+found+i; + new->count=a->count+b->count; + bits_packed+=(uint) (new->count & 7); + bytes_packed+=new->count/8; + new->a.nod.left=a; /* lesser in left */ + new->a.nod.right=b; + queue.root[1]=(byte*) new; + queue_replaced(&queue); + } + huff_tree->root=(HUFF_ELEMENT*) queue.root[1]; + huff_tree->bytes_packed=bytes_packed+(bits_packed+7)/8; + return 0; +} + +static int compare_tree(register const uchar *s, register const uchar *t) +{ + uint length; + for (length=global_count->field_length; length-- ;) + if (*s++ != *t++) + return (int) s[-1] - (int) t[-1]; + return 0; +} + + /* Used by make_huff_tree to save intervall-counts in queue */ + +static int save_counts_in_queue(byte *key, element_count count, + HUFF_TREE *tree) +{ + HUFF_ELEMENT *new; + + new=tree->element_buffer+(tree->elements++); + new->count=count; + new->a.leaf.null=0; + new->a.leaf.element_nr= (uint) (key- tree->counts->tree_buff) / + tree->counts->field_length; + queue.root[tree->elements]=(byte*) new; + return 0; +} + + + /* Calculate length of file if given counts should be used */ + /* Its actually a faster version of make_huff_tree */ + +static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, + uint add_tree_lenght) +{ + uint i,found,bits_packed,first,last; + my_off_t bytes_packed; + HUFF_ELEMENT element_buffer[256]; + DBUG_ENTER("calc_packed_length"); + + first=last=0; + for (i=found=0 ; i < 256 ; i++) + { + if (huff_counts->counts[i]) + { + if (! found++) + first=i; + last=i; + queue.root[found]=(byte*) &huff_counts->counts[i]; + } + } + if (!found) + DBUG_RETURN(0); /* Empty tree */ + if (found < 2) + queue.root[++found]=(byte*) &huff_counts->counts[last ? 0 : 1]; + + queue.elements=found; + + bytes_packed=0; bits_packed=0; + if (add_tree_lenght) + bytes_packed=(8+9+5+5+(max_bit(last-first)+1)*found+ + (max_bit(found-1)+1+1)*(found-2) +7)/8; + for (i=(found+1)/2 ; i > 0 ; i--) + _downheap(&queue,i); + for (i=0 ; i < found-1 ; i++) + { + HUFF_ELEMENT *a,*b,*new; + a=(HUFF_ELEMENT*) queue_remove(&queue,0); + b=(HUFF_ELEMENT*) queue.root[1]; + new=element_buffer+i; + new->count=a->count+b->count; + bits_packed+=(uint) (new->count & 7); + bytes_packed+=new->count/8; + queue.root[1]=(byte*) new; + queue_replaced(&queue); + } + DBUG_RETURN(bytes_packed+(bits_packed+7)/8); +} + + + /* Remove trees that don't give any compression */ + +static uint join_same_trees(HUFF_COUNTS *huff_counts, uint trees) +{ + uint k,tree_number; + HUFF_COUNTS count,*i,*j,*last_count; + + last_count=huff_counts+trees; + for (tree_number=0, i=huff_counts ; i < last_count ; i++) + { + if (!i->tree->tree_number) + { + i->tree->tree_number= ++tree_number; + if (i->tree_buff) + continue; /* Don't join intervall */ + for (j=i+1 ; j < last_count ; j++) + { + if (! j->tree->tree_number && ! j->tree_buff) + { + for (k=0 ; k < 256 ; k++) + count.counts[k]=i->counts[k]+j->counts[k]; + if (calc_packed_length(&count,1) <= + i->tree->bytes_packed + j->tree->bytes_packed+ + i->tree->tree_pack_length+j->tree->tree_pack_length+ + ALLOWED_JOIN_DIFF) + { + memcpy_fixed((byte*) i->counts,(byte*) count.counts, + sizeof(count.counts[0])*256); + my_free((gptr) j->tree->element_buffer,MYF(0)); + j->tree->element_buffer=0; + j->tree=i->tree; + bmove((byte*) i->counts,(byte*) count.counts, + sizeof(count.counts[0])*256); + if (make_huff_tree(i->tree,i)) + return (uint) -1; + } + } + } + } + } + if (verbose) + printf("Original trees: %d After join: %d\n",trees,tree_number); + return tree_number; /* Return trees left */ +} + + + /* Fill in huff_tree decode tables */ + +static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) +{ + uint elements; + for ( ; trees-- ; huff_tree++) + { + if (huff_tree->tree_number > 0) + { + elements=huff_tree->counts->tree_buff ? huff_tree->elements : 256; + if (!(huff_tree->code = + (ulong*) my_malloc(elements* + (sizeof(ulong)+sizeof(uchar)), + MYF(MY_WME | MY_ZEROFILL)))) + return 1; + huff_tree->code_len=(uchar*) (huff_tree->code+elements); + make_traverse_code_tree(huff_tree,huff_tree->root,32,0); + } + } + return 0; +} + + +static void make_traverse_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element, + uint size, ulong code) +{ + uint chr; + if (!element->a.leaf.null) + { + chr=element->a.leaf.element_nr; + huff_tree->code_len[chr]=(uchar) (32-size); + huff_tree->code[chr]= (code >> size); + if (huff_tree->height < 32-size) + huff_tree->height= 32-size; + } + else + { + size--; + make_traverse_code_tree(huff_tree,element->a.nod.left,size,code); + make_traverse_code_tree(huff_tree,element->a.nod.right,size, + code+((ulong) 1L << size)); + } + return; +} + + + /* Write header to new packed data file */ + +static int write_header(MRG_INFO *mrg,uint head_length,uint trees, + my_off_t tot_elements,my_off_t filelength) +{ + byte *buff=file_buffer.pos; + + bzero(buff,HEAD_LENGTH); + memcpy_fixed(buff,myisam_pack_file_magic,4); + int4store(buff+4,head_length); + int4store(buff+8, mrg->min_pack_length); + int4store(buff+12,mrg->max_pack_length); + int4store(buff+16,tot_elements); + int4store(buff+20,intervall_length); + int2store(buff+24,trees); + buff[26]=(char) mrg->ref_length; + /* Save record pointer length */ + buff[27]= (uchar) mi_get_pointer_length((ulonglong) filelength,2); + if (test_only) + return 0; + VOID(my_seek(file_buffer.file,0L,MY_SEEK_SET,MYF(0))); + return my_write(file_buffer.file,file_buffer.pos,HEAD_LENGTH, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; +} + + /* Write fieldinfo to new packed file */ + +static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) +{ + reg1 uint i; + uint huff_tree_bits; + huff_tree_bits=max_bit(trees ? trees-1 : 0); + + for (i=0 ; i++ < fields ; counts++) + { + write_bits((ulong) (int) counts->field_type,5); + write_bits(counts->pack_type,6); + if (counts->pack_type & PACK_TYPE_ZERO_FILL) + write_bits(counts->max_zero_fill,5); + else + write_bits(counts->length_bits,5); + write_bits((ulong) counts->tree->tree_number-1,huff_tree_bits); + } + flush_bits(); + return; +} + + /* Write all huff_trees to new datafile. Return tot count of + elements in all trees + Returns 0 on error */ + +static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) +{ + uint i,int_length; + uint *packed_tree,*offset,length; + my_off_t elements; + + for (i=length=0 ; i < trees ; i++) + if (huff_tree[i].tree_number > 0 && huff_tree[i].elements > length) + length=huff_tree[i].elements; + if (!(packed_tree=(uint*) my_alloca(sizeof(uint)*length*2))) + { + my_error(EE_OUTOFMEMORY,MYF(ME_BELL),sizeof(uint)*length*2); + return 0; + } + + intervall_length=0; + for (elements=0; trees-- ; huff_tree++) + { + if (huff_tree->tree_number == 0) + continue; /* Deleted tree */ + elements+=huff_tree->elements; + huff_tree->max_offset=2; + if (huff_tree->elements <= 1) + offset=packed_tree; + else + offset=make_offset_code_tree(huff_tree,huff_tree->root,packed_tree); + huff_tree->offset_bits=max_bit(huff_tree->max_offset); + if (huff_tree->max_offset >= IS_OFFSET) + { /* This should be impossible */ + VOID(fprintf(stderr,"Tree offset got too big: %d, aborted\n", + huff_tree->max_offset)); + my_afree((gptr) packed_tree); + return 0; + } + +#ifdef EXTRA_DBUG + printf("pos: %d elements: %d tree-elements: %d char_bits: %d\n", + (uint) (file_buffer.pos-file_buffer.buffer), + huff_tree->elements, (offset-packed_tree),huff_tree->char_bits); +#endif + if (!huff_tree->counts->tree_buff) + { + write_bits(0,1); + write_bits(huff_tree->min_chr,8); + write_bits(huff_tree->elements,9); + write_bits(huff_tree->char_bits,5); + write_bits(huff_tree->offset_bits,5); + int_length=0; + } + else + { + int_length=(uint) (huff_tree->counts->tree_pos - + huff_tree->counts->tree_buff); + write_bits(1,1); + write_bits(huff_tree->elements,15); + write_bits(int_length,16); + write_bits(huff_tree->char_bits,5); + write_bits(huff_tree->offset_bits,5); + intervall_length+=int_length; + } + length=(uint) (offset-packed_tree); + if (length != huff_tree->elements*2-2) + printf("error: Huff-tree-length: %d != calc_length: %d\n", + length,huff_tree->elements*2-2); + + for (i=0 ; i < length ; i++) + { + if (packed_tree[i] & IS_OFFSET) + write_bits(packed_tree[i] - IS_OFFSET+ (1 << huff_tree->offset_bits), + huff_tree->offset_bits+1); + else + write_bits(packed_tree[i]-huff_tree->min_chr,huff_tree->char_bits+1); + } + flush_bits(); + if (huff_tree->counts->tree_buff) + { + for (i=0 ; i < int_length ; i++) + write_bits((uint) (uchar) huff_tree->counts->tree_buff[i],8); + } + flush_bits(); + } + my_afree((gptr) packed_tree); + return elements; +} + + +static uint *make_offset_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, + uint *offset) +{ + uint *prev_offset; + + prev_offset= offset; + if (!element->a.nod.left->a.leaf.null) + { + offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; + offset+=2; + } + else + { + prev_offset[0]= IS_OFFSET+2; + offset=make_offset_code_tree(huff_tree,element->a.nod.left,offset+2); + } + if (!element->a.nod.right->a.leaf.null) + { + prev_offset[1]=element->a.nod.right->a.leaf.element_nr; + return offset; + } + else + { + uint temp=(uint) (offset-prev_offset-1); + prev_offset[1]= IS_OFFSET+ temp; + if (huff_tree->max_offset < temp) + huff_tree->max_offset = temp; + return make_offset_code_tree(huff_tree,element->a.nod.right,offset); + } +} + + /* Get number of bits neaded to represent value */ + +static uint max_bit(register uint value) +{ + reg2 uint power=1; + + while ((value>>=1)) + power++; + return (power); +} + + +static int compress_isam_file(MRG_INFO *mrg, HUFF_COUNTS *huff_counts) +{ + int error; + uint i,max_calc_length,pack_ref_length,min_record_length,max_record_length, + intervall,field_length,max_pack_length,pack_blob_length; + my_off_t record_count; + ulong length,pack_length; + byte *record,*pos,*end_pos,*record_pos,*start_pos; + HUFF_COUNTS *count,*end_count; + HUFF_TREE *tree; + MI_INFO *isam_file=mrg->file[0]; + DBUG_ENTER("compress_isam_file"); + + if (!(record=(byte*) my_alloca(isam_file->s->base.reclength))) + return -1; + end_count=huff_counts+isam_file->s->base.fields; + min_record_length= (uint) ~0; + max_record_length=0; + + for (i=max_calc_length=0 ; i < isam_file->s->base.fields ; i++) + { + if (!(huff_counts[i].pack_type & PACK_TYPE_ZERO_FILL)) + huff_counts[i].max_zero_fill=0; + if (huff_counts[i].field_type == FIELD_CONSTANT || + huff_counts[i].field_type == FIELD_ZERO || + huff_counts[i].field_type == FIELD_CHECK) + continue; + if (huff_counts[i].field_type == FIELD_INTERVALL) + max_calc_length+=huff_counts[i].tree->height; + else if (huff_counts[i].field_type == FIELD_BLOB || + huff_counts[i].field_type == FIELD_VARCHAR) + max_calc_length=huff_counts[i].tree->height*huff_counts[i].max_length + huff_counts[i].length_bits +1; + else + max_calc_length+= + (huff_counts[i].field_length - huff_counts[i].max_zero_fill)* + huff_counts[i].tree->height+huff_counts[i].length_bits; + } + max_calc_length/=8; + if (max_calc_length < 254) + pack_ref_length=1; + else if (max_calc_length <= 65535) + pack_ref_length=3; + else + pack_ref_length=4; + record_count=0; + pack_blob_length=0; + if (isam_file->s->base.blobs) + { + if (mrg->max_blob_length < 254) + pack_blob_length=1; + else if (mrg->max_blob_length <= 65535) + pack_blob_length=3; + else + pack_blob_length=4; + } + max_pack_length=pack_ref_length+pack_blob_length; + + mrg_reset(mrg); + while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) + { + ulong tot_blob_length=0; + if (! error) + { + if (flush_buffer(max_calc_length+max_pack_length)) + break; + record_pos=file_buffer.pos; + file_buffer.pos+=max_pack_length; + for (start_pos=record, count= huff_counts; count < end_count ; count++) + { + end_pos=start_pos+(field_length=count->field_length); + tree=count->tree; + + if (count->pack_type & PACK_TYPE_SPACE_FIELDS) + { + for (pos=start_pos ; *pos == ' ' && pos < end_pos; pos++) ; + if (pos == end_pos) + { + write_bits(1,1); + start_pos=end_pos; + continue; + } + write_bits(0,1); + } + end_pos-=count->max_zero_fill; + field_length-=count->max_zero_fill; + + switch(count->field_type) { + case FIELD_SKIPP_ZERO: + if (!memcmp((byte*) start_pos,zero_string,field_length)) + { + write_bits(1,1); + start_pos=end_pos; + break; + } + write_bits(0,1); + /* Fall through */ + case FIELD_NORMAL: + for ( ; start_pos < end_pos ; start_pos++) + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + break; + case FIELD_SKIPP_ENDSPACE: + for (pos=end_pos ; pos > start_pos && pos[-1] == ' ' ; pos--) ; + length=(uint) (end_pos-pos); + if (count->pack_type & PACK_TYPE_SELECTED) + { + if (length > count->min_space) + { + write_bits(1,1); + write_bits(length,count->length_bits); + } + else + { + write_bits(0,1); + pos=end_pos; + } + } + else + write_bits(length,count->length_bits); + for ( ; start_pos < pos ; start_pos++) + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + start_pos=end_pos; + break; + case FIELD_SKIPP_PRESPACE: + for (pos=start_pos ; pos < end_pos && pos[0] == ' ' ; pos++) ; + length=(uint) (pos-start_pos); + if (count->pack_type & PACK_TYPE_SELECTED) + { + if (length > count->min_space) + { + write_bits(1,1); + write_bits(length,count->length_bits); + } + else + { + pos=start_pos; + write_bits(0,1); + } + } + else + write_bits(length,count->length_bits); + for (start_pos=pos ; start_pos < end_pos ; start_pos++) + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + break; + case FIELD_CONSTANT: + case FIELD_ZERO: + case FIELD_CHECK: + start_pos=end_pos; + break; + case FIELD_INTERVALL: + global_count=count; + pos=(byte*) tree_search(&count->int_tree,start_pos); + intervall=(uint) (pos - count->tree_buff)/field_length; + write_bits(tree->code[intervall],(uint) tree->code_len[intervall]); + start_pos=end_pos; + break; + case FIELD_BLOB: + { + ulong blob_length=_mi_calc_blob_length(field_length- + mi_portable_sizeof_char_ptr, + start_pos); + if (!blob_length) + { + write_bits(1,1); /* Empty blob */ + } + else + { + byte *blob,*blob_end; + write_bits(0,1); + write_bits(blob_length,count->length_bits); + memcpy_fixed(&blob,end_pos-mi_portable_sizeof_char_ptr, + sizeof(char*)); + blob_end=blob+blob_length; + for ( ; blob < blob_end ; blob++) + write_bits(tree->code[(uchar) *blob], + (uint) tree->code_len[(uchar) *blob]); + tot_blob_length+=blob_length; + } + start_pos= end_pos; + break; + } + case FIELD_VARCHAR: + { + ulong col_length= uint2korr(start_pos); + if (!col_length) + { + write_bits(1,1); /* Empty varchar */ + } + else + { + byte *end=start_pos+2+col_length; + write_bits(0,1); + write_bits(col_length,count->length_bits); + for (start_pos+=2 ; start_pos < end ; start_pos++) + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + } + start_pos= end_pos; + break; + } + case FIELD_LAST: + abort(); /* Impossible */ + } + start_pos+=count->max_zero_fill; + } + flush_bits(); + length=(ulong) (file_buffer.pos-record_pos)-max_pack_length; + pack_length=save_pack_length(record_pos,length); + if (pack_blob_length) + pack_length+=save_pack_length(record_pos+pack_length,tot_blob_length); + + /* Correct file buffer if the header was smaller */ + if (pack_length != max_pack_length) + { + bmove(record_pos+pack_length,record_pos+max_pack_length,length); + file_buffer.pos-= (max_pack_length-pack_length); + } + if (length < (ulong) min_record_length) + min_record_length=(uint) length; + if (length > (ulong) max_record_length) + max_record_length=(uint) length; + if (write_loop && ++record_count % WRITE_COUNT == 0) + { + printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + } + } + else if (error != HA_ERR_RECORD_DELETED) + break; + } + if (error == HA_ERR_END_OF_FILE) + error=0; + else + { + fprintf(stderr,"%s: Got error %d reading records\n",my_progname,error); + } + + my_afree((gptr) record); + mrg->ref_length=max_pack_length; + mrg->min_pack_length=max_record_length ? min_record_length : 0; + mrg->max_pack_length=max_record_length; + DBUG_RETURN(error || error_on_write || flush_buffer(~(ulong) 0)); +} + + +static char *make_new_name(char *new_name, char *old_name) +{ + return fn_format(new_name,old_name,"",DATA_TMP_EXT,2+4); +} + +static char *make_old_name(char *new_name, char *old_name) +{ + return fn_format(new_name,old_name,"",OLD_EXT,2+4); +} + + /* rutines for bit writing buffer */ + +static void init_file_buffer(File file, pbool read_buffer) +{ + file_buffer.file=file; + file_buffer.buffer=my_malloc(ALIGN_SIZE(RECORD_CACHE_SIZE),MYF(MY_WME)); + file_buffer.end=file_buffer.buffer+ALIGN_SIZE(RECORD_CACHE_SIZE)-8; + file_buffer.pos_in_file=0; + error_on_write=0; + if (read_buffer) + { + + file_buffer.pos=file_buffer.end; + file_buffer.bits=0; + } + else + { + file_buffer.pos=file_buffer.buffer; + file_buffer.bits=BITS_SAVED; + } + file_buffer.byte=0; +} + + +static int flush_buffer(ulong neaded_length) +{ + ulong length; + if ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length) + return 0; + length=(ulong) (file_buffer.pos-file_buffer.buffer); + file_buffer.pos=file_buffer.buffer; + file_buffer.pos_in_file+=length; + if (test_only) + return 0; + if (error_on_write|| my_write(file_buffer.file,file_buffer.buffer, + length, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL))) + { + error_on_write=1; + return 1; + } + + if (neaded_length != ~(ulong) 0 && + (ulong) (file_buffer.end-file_buffer.buffer) < neaded_length) + { + char *tmp; + neaded_length+=256; /* some margin */ + tmp=my_realloc(file_buffer.buffer, neaded_length,MYF(MY_WME)); + if (!tmp) + return 1; + file_buffer.pos= tmp + (ulong) (file_buffer.pos - file_buffer.buffer); + file_buffer.buffer=tmp; + file_buffer.end=tmp+neaded_length-8; + } + return 0; +} + + +static void end_file_buffer(void) +{ + my_free((gptr) file_buffer.buffer,MYF(0)); +} + + /* output `bits` low bits of `value' */ + +static void write_bits (register ulong value, register uint bits) +{ + if ((file_buffer.bits-=(int) bits) >= 0) + { + file_buffer.byte|=value << file_buffer.bits; + } + else + { + reg3 uint byte_buff; + bits= (uint) -file_buffer.bits; + byte_buff=file_buffer.byte | (uint) (value >> bits); +#if BITS_SAVED == 32 + *file_buffer.pos++= (byte) (byte_buff >> 24) ; + *file_buffer.pos++= (byte) (byte_buff >> 16) ; +#endif + *file_buffer.pos++= (byte) (byte_buff >> 8) ; + *file_buffer.pos++= (byte) byte_buff; + + value&=(1 << bits)-1; +#if BITS_SAVED == 16 + if (bits >= sizeof(uint)) + { + bits-=8; + *file_buffer.pos++= (uchar) (value >> bits); + value&= (1 << bits)-1; + if (bits >= sizeof(uint)) + { + bits-=8; + *file_buffer.pos++= (uchar) (value >> bits); + value&= (1 << bits)-1; + } + } +#endif + if (file_buffer.pos >= file_buffer.end) + VOID(flush_buffer((uint) ~0)); + file_buffer.bits=(int) (BITS_SAVED - bits); + file_buffer.byte=(uint) (value << (BITS_SAVED - bits)); + } + return; +} + + /* Flush bits in bit_buffer to buffer */ + +static void flush_bits (void) +{ + uint bits,byte_buff; + + bits=(file_buffer.bits) & ~7; + byte_buff = file_buffer.byte >> bits; + bits=BITS_SAVED - bits; + while (bits > 0) + { + bits-=8; + *file_buffer.pos++= (byte) (uchar) (byte_buff >> bits) ; + } + file_buffer.bits=BITS_SAVED; + file_buffer.byte=0; + return; +} + + +/**************************************************************************** +** functions to handle the joined files +****************************************************************************/ + +static int save_state(MI_INFO *isam_file,MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc) +{ + MYISAM_SHARE *share=isam_file->s; + uint options=mi_uint2korr(share->state.header.options); + uint key; + DBUG_ENTER("save_state"); + + options|= HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA; + mi_int2store(share->state.header.options,options); + + share->state.state.data_file_length=new_length; + share->state.state.del=0; + share->state.state.empty=0; + share->state.dellink= HA_OFFSET_ERROR; + share->state.split=(ha_rows) mrg->records; + share->state.version=(ulong) time((time_t*) 0); + share->state.key_map=0; + share->state.state.key_file_length=share->base.keystart; + for (key=0 ; key < share->base.keys ; key++) + share->state.key_root[key]= HA_OFFSET_ERROR; + for (key=0 ; key < share->state.header.max_block_size ; key++) + share->state.key_del[key]= HA_OFFSET_ERROR; + share->state.checksum=crc; /* Save crc here */ + share->changed=1; /* Force write of header */ + share->state.open_count=0; + share->global_changed=0; + VOID(my_chsize(share->kfile,share->state.state.key_file_length, + MYF(0))); + if (share->base.keys) + isamchk_neaded=1; + DBUG_RETURN(mi_state_info_write(share->kfile,&share->state,1+2)); +} + + +static int save_state_mrg(File file,MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc) +{ + MI_STATE_INFO state; + MI_INFO *isam_file=mrg->file[0]; + uint options; + DBUG_ENTER("save_state_mrg"); + + state= isam_file->s->state; + options= (mi_uint2korr(state.header.options) | HA_OPTION_COMPRESS_RECORD | + HA_OPTION_READ_ONLY_DATA); + mi_int2store(state.header.options,options); + state.state.data_file_length=new_length; + state.state.del=0; + state.state.empty=0; + state.state.records=state.split=(ha_rows) mrg->records; + state.state.key_file_length=isam_file->s->base.keystart; + state.dellink= HA_OFFSET_ERROR; + state.version=(ulong) time((time_t*) 0); + state.key_map=0; + state.checksum=crc; + if (isam_file->s->base.keys) + isamchk_neaded=1; + state.changed=1; /* Force one check of table */ + DBUG_RETURN (mi_state_info_write(file,&state,1+2)); +} + + +/* reset for mrg_rrnd */ + +static void mrg_reset(MRG_INFO *mrg) +{ + if (mrg->current) + { + mi_extra(*mrg->current,HA_EXTRA_NO_CACHE); + mrg->current=0; + } +} + +static int mrg_rrnd(MRG_INFO *info,byte *buf) +{ + int error; + MI_INFO *isam_info; + my_off_t filepos; + + if (!info->current) + { + isam_info= *(info->current=info->file); + info->end=info->current+info->count; + mi_extra(isam_info,HA_EXTRA_CACHE); + mi_extra(isam_info,HA_EXTRA_RESET); + filepos=isam_info->s->pack.header_length; + } + else + { + isam_info= *info->current; + filepos= isam_info->nextpos; + } + + for (;;) + { + isam_info->update&= HA_STATE_CHANGED; + if (!(error=(*isam_info->s->read_rnd)(isam_info,(byte*) buf, + filepos, 1)) || + error != HA_ERR_END_OF_FILE) + return (error); + mi_extra(isam_info,HA_EXTRA_NO_CACHE); + if (info->current+1 == info->end) + return(HA_ERR_END_OF_FILE); + info->current++; + isam_info= *info->current; + filepos=isam_info->s->pack.header_length; + mi_extra(isam_info,HA_EXTRA_CACHE); + mi_extra(isam_info,HA_EXTRA_RESET); + } +} + + +static int mrg_close(MRG_INFO *mrg) +{ + uint i; + int error=0; + for (i=0 ; i < mrg->count ; i++) + error|=mi_close(mrg->file[i]); + if (mrg->free_file) + my_free((gptr) mrg->file,MYF(0)); + return error; +} diff --git a/myisam/sort.c b/myisam/sort.c new file mode 100644 index 00000000000..0a5b575242f --- /dev/null +++ b/myisam/sort.c @@ -0,0 +1,531 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Creates a index for a database by reading keys, sorting them and outputing + them in sorted order through SORT_INFO functions. +*/ + +#include "myisamdef.h" +#if defined(MSDOS) || defined(__WIN__) +#include <fcntl.h> +#else +#include <stddef.h> +#endif +#include <queues.h> + + /* static variabels */ + +#define MERGEBUFF 15 +#define MERGEBUFF2 31 +#define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD) +#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) + +typedef struct st_buffpek { + my_off_t file_pos; /* position to buffer */ + ha_rows count; /* keys in buffer */ + uchar *base,*key; /* Pekare inom sort_key - indexdel */ + uint mem_count; /* keys left in memory */ + uint max_keys; /* Max keys in buffert */ +} BUFFPEK; + +extern void print_error _VARARGS((const char *fmt,...)); + + /* functions defined in this file */ + +static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info,uint keys, + uchar * *sort_keys, + BUFFPEK *buffpek,int *maxbuffer, + FILE **tempfile, my_string tempname); +static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar * *sort_keys, + uint count, BUFFPEK *buffpek,FILE **tempfile, + my_string tempname); +static int NEAR_F write_index(MI_SORT_PARAM *info,uchar * *sort_keys, + uint count); +static int NEAR_F merge_many_buff(MI_SORT_PARAM *info,uint keys, + uchar * *sort_keys, + BUFFPEK *buffpek,int *maxbuffer, + FILE * *t_file, my_string tempname); +static uint NEAR_F read_to_buffer(FILE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys,FILE *from_file, + FILE *to_file, uchar * *sort_keys, + BUFFPEK *lastbuff,BUFFPEK *Fb, + BUFFPEK *Tb); +static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, + FILE *); +static char **make_char_array(uint fields,uint length,myf my_flag); +static FILE *opentemp(my_string name,const char *temp_dir); +static void closetemp(my_string name,FILE *stream); + + + /* Creates a index of sorted keys */ + /* Returns 0 if everything went ok */ + +int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, + ulong sortbuff_size) +{ + int error,maxbuffer,skr; + uint memavl,old_memavl,keys,sort_length; + BUFFPEK *buffpek; + char tempname[FN_REFLEN]; + ha_rows records; + uchar **sort_keys; + FILE *tempfile; + DBUG_ENTER("_create_index_by_sort"); + DBUG_PRINT("enter",("sort_length: %d", info->key_length)); + + tempfile=0; buffpek= (BUFFPEK *) NULL; sort_keys= (uchar **) NULL; error= 1; + maxbuffer=1; + + memavl=max(sortbuff_size,MIN_SORT_MEMORY); + records= info->max_records; + sort_length= info->key_length; + LINT_INIT(keys); + + while (memavl >= MIN_SORT_MEMORY) + { + if ((my_off_t) (records+1)*(sort_length+sizeof(char*)) <= + (my_off_t) memavl) + keys= records+1; + else + do + { + skr=maxbuffer; + if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer || + (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/ + (sort_length+sizeof(char*))) <= 1) + { + mi_check_print_error(info->sort_info->param, + "sort_buffer_size is to small"); + goto err; + } + } + while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); + + if ((sort_keys= (uchar **) make_char_array(keys,sort_length,MYF(0)))) + { + if ((buffpek = (BUFFPEK*) my_malloc((uint) (sizeof(BUFFPEK)* + (uint) maxbuffer), + MYF(0)))) + break; + else + my_free((gptr) sort_keys,MYF(0)); + } + old_memavl=memavl; + if ((memavl=memavl/4*3) < MIN_SORT_MEMORY && old_memavl > MIN_SORT_MEMORY) + memavl=MIN_SORT_MEMORY; + } + if (memavl < MIN_SORT_MEMORY) + { + mi_check_print_error(info->sort_info->param,"Sort buffer to small"); + goto err; + } + (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ + + if (!no_messages) + printf(" - Searching for keys, allocating buffer for %d keys\n",keys); + + if ((records=find_all_keys(info,keys,sort_keys,buffpek,&maxbuffer,&tempfile, + tempname)) + == HA_POS_ERROR) + goto err; + if (maxbuffer == 0) + { + if (!no_messages) + printf(" - Dumping %lu keys\n",records); + if (write_index(info,sort_keys,(uint) records)) + goto err; + } + else + { + keys=(keys*(sort_length+sizeof(char*)))/sort_length; + if (maxbuffer >= MERGEBUFF2) + { + if (!no_messages) + printf(" - Merging %lu keys\n",records); + if (merge_many_buff(info,keys,sort_keys,buffpek,&maxbuffer,&tempfile, + tempname)) + goto err; + } + if (!no_messages) + puts(" - Last merge and dumping keys"); + if (merge_index(info,keys,sort_keys,buffpek,maxbuffer,tempfile)) + goto err; + } + error =0; + +err: + if (sort_keys) + my_free((gptr) sort_keys,MYF(0)); + if (buffpek) + my_free((gptr) buffpek,MYF(0)); + if (tempfile) + closetemp(tempname,tempfile); + + DBUG_RETURN(error ? -1 : 0); +} /* _create_index_by_sort */ + + + /* Search after all keys and place them in a temp. file */ + +static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, + uchar **sort_keys, BUFFPEK *buffpek, + int *maxbuffer, FILE **tempfile, + my_string tempname) +{ + int error; + uint idx,indexpos; + DBUG_ENTER("find_all_keys"); + + idx=indexpos=error=0; + + while (!(error=(*info->key_read)(info->sort_info,sort_keys[idx]))) + { + if ((uint) ++idx == keys) + { + if (indexpos >= (uint) *maxbuffer || + write_keys(info,sort_keys,idx-1,buffpek+indexpos,tempfile, + tempname)) + DBUG_RETURN((ha_rows) -1); + memcpy(sort_keys[0],sort_keys[idx-1],(size_t) info->key_length); + idx=1; indexpos++; + } + } + if (error > 0) + DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ + if (indexpos) + if (indexpos >= (uint) *maxbuffer || + write_keys(info,sort_keys,idx,buffpek+indexpos,tempfile,tempname)) + DBUG_RETURN(HA_POS_ERROR); + *maxbuffer=(int) indexpos; + DBUG_RETURN(indexpos*(keys-1)+idx); +} /* find_all_keys */ + + + /* Write all keys in memory to file for later merge */ + +static int NEAR_F write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, + uint count, BUFFPEK *buffpek, + register FILE **tempfile, my_string tempname) +{ + DBUG_ENTER("write_keys"); + + qsort2((byte*) sort_keys,count,sizeof(byte*),(qsort2_cmp) info->key_cmp, + info->sort_info); + if (! *tempfile && ! (*tempfile=opentemp(tempname,info->tmpdir))) + DBUG_RETURN(1); + buffpek->file_pos=my_ftell(*tempfile,MYF(0)); + buffpek->count=count; + while (count--) + if (my_fwrite(*tempfile,(byte*)*sort_keys++,info->key_length,MYF_RW)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} /* write_keys */ + + + /* Write index */ + +static int NEAR_F write_index(MI_SORT_PARAM *info, register uchar **sort_keys, register uint count) +{ + DBUG_ENTER("write_index"); + + qsort2((gptr) sort_keys,(size_t) count,sizeof(byte*), + (qsort2_cmp) info->key_cmp,info->sort_info); + while (count--) + if ((*info->key_write)(info->sort_info,*sort_keys++)) + DBUG_RETURN(-1); + DBUG_RETURN(0); +} /* write_index */ + + + /* Merge buffers to make < MERGEBUFF2 buffers */ + +static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, + uchar **sort_keys, BUFFPEK *buffpek, + int *maxbuffer, FILE **t_file, + my_string t_name) +{ + register int i; + FILE *from_file,*to_file,*temp; + FILE *t_file2; + char t_name2[FN_REFLEN]; + BUFFPEK *lastbuff; + DBUG_ENTER("merge_many_buff"); + + if (!(t_file2=opentemp(t_name2,info->tmpdir))) + DBUG_RETURN(1); + + from_file= *t_file ; to_file= t_file2; + while (*maxbuffer >= MERGEBUFF2) + { + lastbuff=buffpek; + for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) + { + if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, + buffpek+i,buffpek+i+MERGEBUFF-1)) + break; + } + if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, + buffpek+i,buffpek+ *maxbuffer)) + break; + *maxbuffer= (int) (lastbuff-buffpek)-1; + temp=from_file; from_file=to_file; to_file=temp; + VOID(my_fseek(to_file,0L,MY_SEEK_SET,MYF(0))); + } + if (to_file == *t_file) + { + closetemp(t_name,to_file); + *t_file=t_file2; + VOID(strmov(t_name,t_name2)); + } + else closetemp(t_name2,to_file); + + DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ +} /* merge_many_buff */ + + + /* Read data to buffer */ + /* This returns (uint) -1 if something goes wrong */ + +static uint NEAR_F read_to_buffer(FILE *fromfile, BUFFPEK *buffpek, uint sort_length) +{ + register uint count; + uint length; + + count=buffpek->max_keys; + if ((ha_rows) count > buffpek->count) + count=(uint) buffpek->count; + if (count) + { + VOID(my_fseek(fromfile,buffpek->file_pos,MY_SEEK_SET,MYF(0))); + if (my_fread(fromfile,(byte*) buffpek->base, + (length= sort_length*count),MYF_RW)) + return((uint) -1); + buffpek->key=buffpek->base; + buffpek->file_pos+= length; /* New filepos */ + buffpek->count-= count; + buffpek->mem_count= count; + } + return (count*sort_length); +} /* read_to_buffer */ + + + /* Merge buffers to one buffer */ + /* If to_file == 0 then use info->key_write */ + +static int NEAR_F +merge_buffers(MI_SORT_PARAM *info, uint keys, FILE *from_file, FILE *to_file, + uchar **sort_keys, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb) +{ + int error; + uint sort_length,maxcount; + ha_rows count; + my_off_t to_start_filepos; + uchar *strpos; + BUFFPEK *buffpek,**refpek; + QUEUE queue; + DBUG_ENTER("merge_buffers"); + + count=error=0; + maxcount=keys/((uint) (Tb-Fb) +1); + sort_length=info->key_length; + + LINT_INIT(to_start_filepos); + if (to_file) + to_start_filepos=my_ftell(to_file,MYF(0)); + strpos=(uchar*) sort_keys; + + if (init_queue(&queue,(uint) (Tb-Fb)+1,offsetof(BUFFPEK,key),0, + (int (*)(void*, byte *,byte*)) info->key_cmp, + (void*) info->sort_info)) + DBUG_RETURN(1); + + for (buffpek= Fb ; buffpek <= Tb && error != -1 ; buffpek++) + { + count+= buffpek->count; + buffpek->base= strpos; + buffpek->max_keys=maxcount; + strpos+= (uint) (error=(int) read_to_buffer(from_file,buffpek, + sort_length)); + queue_insert(&queue,(void*) buffpek); + } + if (error == -1) + goto err; + + while (queue.elements > 1) + { + for (;;) + { + buffpek=(BUFFPEK*) queue_top(&queue); + if (to_file) + { + if (my_fwrite(to_file,(byte*) buffpek->key,(uint) sort_length, + MYF_RW | MY_WAIT_IF_FULL)) + { + error=1; goto err; + } + } + else + { + if ((*info->key_write)(info->sort_info,(void*) buffpek->key)) + { + error=1; goto err; + } + } + buffpek->key+=sort_length; + if (! --buffpek->mem_count) + { + if (!(error=(int) read_to_buffer(from_file,buffpek,sort_length))) + { + uchar *base=buffpek->base; + uint max_keys=buffpek->max_keys; + + VOID(queue_remove(&queue,0)); + + /* Put room used by buffer to use in other buffer */ + for (refpek= (BUFFPEK**) &queue_top(&queue); + refpek <= (BUFFPEK**) &queue_end(&queue); + refpek++) + { + buffpek= *refpek; + if (buffpek->base+buffpek->max_keys*sort_length == base) + { + buffpek->max_keys+=max_keys; + break; + } + else if (base+max_keys*sort_length == buffpek->base) + { + buffpek->base=base; + buffpek->max_keys+=max_keys; + break; + } + } + break; /* One buffer have been removed */ + } + } + queue_replaced(&queue); /* Top element has been replaced */ + } + } + buffpek=(BUFFPEK*) queue_top(&queue); + buffpek->base=(uchar *) sort_keys; + buffpek->max_keys=keys; + do + { + if (to_file) + { + if (my_fwrite(to_file,(byte*) buffpek->key, + (uint) (sort_length*buffpek->mem_count), + MYF_RW | MY_WAIT_IF_FULL)) + { + error=1; goto err; + } + } + else + { + register uchar *end; + strpos= buffpek->key; + for (end=strpos+buffpek->mem_count*sort_length; + strpos != end ; + strpos+=sort_length) + { + if ((*info->key_write)(info->sort_info,(void*) strpos)) + { + error=1; goto err; + } + } + } + } + while ((error=(int) read_to_buffer(from_file,buffpek,sort_length)) != -1 && + error != 0); + + lastbuff->count=count; + if (to_file) + lastbuff->file_pos=to_start_filepos; /* New block starts here */ +err: + delete_queue(&queue); + DBUG_RETURN(error); +} /* merge_buffers */ + + + /* Do a merge to output-file (save only positions) */ + +static int NEAR_F +merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, + BUFFPEK *buffpek, int maxbuffer, FILE *tempfile) +{ + DBUG_ENTER("merge_index"); + if (merge_buffers(info,keys,tempfile,(FILE*) 0,sort_keys,buffpek,buffpek, + buffpek+maxbuffer)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} /* merge_index */ + + + /* Make a pointer of arrays to keys */ + +static char **make_char_array(register uint fields, uint length, myf my_flag) +{ + register char **pos; + char **old_pos,*char_pos; + DBUG_ENTER("make_char_array"); + + if ((old_pos= (char**) my_malloc( fields*(length+sizeof(char*)), my_flag))) + { + pos=old_pos; char_pos=((char*) (pos+fields)) -length; + while (fields--) + *(pos++) = (char_pos+= length); + } + + DBUG_RETURN(old_pos); +} /* make_char_array */ + + +/* Open a temporary file that will be deleted on close */ + +static FILE *opentemp(my_string name,const char *temp_dir) +{ + FILE *stream; + reg1 my_string str_pos; + DBUG_ENTER("opentemp"); + + if (!(str_pos=my_tempnam(temp_dir,"ST",MYF(MY_WME)))) + DBUG_RETURN(0); + VOID(strmov(name,str_pos)); + (*free)(str_pos); /* Avoid the 'free' macro */ + + stream=my_fopen(name,(int) (O_RDWR | FILE_BINARY | O_CREAT | O_TEMPORARY), + MYF(MY_WME)); +#if O_TEMPORARY == 0 && !defined(CANT_DELETE_OPEN_FILES) + VOID(my_delete(name,MYF(MY_WME | ME_NOINPUT))); +#endif + DBUG_PRINT("exit",("stream: %lx",stream)); + DBUG_RETURN (stream); +} /* opentemp */ + + +static void closetemp(my_string name __attribute__((unused)), FILE *stream) +{ + DBUG_ENTER("closetemp"); + + if (stream) + VOID(my_fclose(stream,MYF(MY_WME))); +#if !(O_TEMPORARY == 0 && !defined(CANT_DELETE_OPEN_FILES)) + if (name) + VOID(my_delete(name,MYF(MY_WME))); +#endif + DBUG_VOID_RETURN; +} /* closetemp */ diff --git a/myisam/test_pack b/myisam/test_pack new file mode 100755 index 00000000000..0cbeb57ba70 --- /dev/null +++ b/myisam/test_pack @@ -0,0 +1,9 @@ +silent="-s" +suffix=$MACH +mi_test1$MACH -s ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -us test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -S ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 ;myisamchk$MACH -us test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -b ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -w ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -ros test1 ; myisamchk$MACH -es test1 + +mi_test2$MACH -s -t4 ; myisampack$MACH --force -s test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -ros test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -s -u test2 ; myisamchk$MACH -sm test2 +mi_test2$MACH -s -t4 -b -N ; myisampack$MACH --force -s test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -ros test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -s -u test2 ; myisamchk$MACH -sm test2 |