diff options
author | Michael Widenius <monty@askmonty.org> | 2011-07-24 01:25:28 -0700 |
---|---|---|
committer | Michael Widenius <monty@askmonty.org> | 2011-07-24 01:25:28 -0700 |
commit | 10065404eae16de02aa843a52d1d1f48dd23e9ac (patch) | |
tree | f4fb6c0dfe883042482babbcb98df5f0a4d25788 /storage/maria | |
parent | 3c97c9c9cc3dde2249846c35020d2eaf2af1bcd0 (diff) | |
download | mariadb-git-10065404eae16de02aa843a52d1d1f48dd23e9ac.tar.gz |
Fixes lp:805930 Sysbench breaks on multiple table test with MariaDB 5.2.7 + Aria
The bug happens when one uses MAX_ROWS=# with Aria & row_format=page and one insert more than # rows.
mysql-test/mysql-test-run.pl:
Ignore table is full error messages
mysql-test/suite/maria/r/max_length.result:
Test case for 'Table is full'
mysql-test/suite/maria/t/max_length.test:
Test case for 'Table is full'
storage/maria/ma_bitmap.c:
Ensure that we don't allocate bits outside of max_data_file_size.
Adjust max_data_file_size based on bitmap alignments.
Backport fix to adjust wrong first_bitmap_with_space.
storage/maria/ma_blockrec.c:
Calculate value of max_data_file_length
storage/maria/ma_blockrec.h:
Updated prototype for _ma_bitmap_init()
storage/maria/ma_check.c:
Give warnings if file sizes are above max file sizes.
Give more warnings in case of errors.
Have maria_chk write if table is recreated.
storage/maria/ma_create.c:
Better calculation of max_data_file_length and thus data pointer length.
Fixes some wrong pointer lengths when using MAX_ROWS=#
storage/maria/ma_open.c:
Removed duplicate assigment.
Use block size from file instead of global variable.
storage/maria/maria_chk.c:
Remove -1 from printed file length
storage/maria/maria_def.h:
Update struct st_maria_file_bitmap
Diffstat (limited to 'storage/maria')
-rw-r--r-- | storage/maria/ma_bitmap.c | 88 | ||||
-rw-r--r-- | storage/maria/ma_blockrec.c | 27 | ||||
-rw-r--r-- | storage/maria/ma_blockrec.h | 3 | ||||
-rw-r--r-- | storage/maria/ma_check.c | 58 | ||||
-rw-r--r-- | storage/maria/ma_create.c | 16 | ||||
-rw-r--r-- | storage/maria/ma_open.c | 4 | ||||
-rw-r--r-- | storage/maria/maria_chk.c | 4 | ||||
-rw-r--r-- | storage/maria/maria_def.h | 5 |
8 files changed, 156 insertions, 49 deletions
diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 96a5e8a5650..bd24ddb35ea 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -209,7 +209,10 @@ static inline my_bool write_changed_bitmap(MARIA_SHARE *share, SYNOPSIS _ma_bitmap_init() share Share handler - file data file handler + file Data file handler + last_page Pointer to last page (max_file_size) that needs to be + mapped by the bitmap. This is adjusted to bitmap + alignment. NOTES This is called the first time a file is opened. @@ -219,12 +222,14 @@ static inline my_bool write_changed_bitmap(MARIA_SHARE *share, 1 error */ -my_bool _ma_bitmap_init(MARIA_SHARE *share, File file) +my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, + pgcache_page_no_t *last_page) { uint aligned_bit_blocks; uint max_page_size; MARIA_FILE_BITMAP *bitmap= &share->bitmap; uint size= share->block_size; + pgcache_page_no_t first_bitmap_with_space; #ifndef DBUG_OFF /* We want to have a copy of the bitmap to be able to print differences */ size*= 2; @@ -241,7 +246,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file) /* Size needs to be aligned on 6 */ aligned_bit_blocks= (share->block_size - PAGE_SUFFIX_SIZE) / 6; - bitmap->total_size= aligned_bit_blocks * 6; + bitmap->max_total_size= bitmap->total_size= aligned_bit_blocks * 6; /* In each 6 bytes, we have 6*8/3 = 16 pages covered The +1 is to add the bitmap page, as this doesn't have to be covered @@ -266,13 +271,34 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file) pthread_mutex_init(&share->bitmap.bitmap_lock, MY_MUTEX_INIT_SLOW); pthread_cond_init(&share->bitmap.bitmap_cond, 0); + first_bitmap_with_space= share->state.first_bitmap_with_space; _ma_bitmap_reset_cache(share); - if (share->state.first_bitmap_with_space == ~(pgcache_page_no_t) 0) - { - /* Start scanning for free space from start of file */ - share->state.first_bitmap_with_space = 0; - } + /* + The bitmap used to map the file are aligned on 6 bytes. We now + calculate the max file size that can be used by the bitmap. This + is needed to get ma_info() give a true file size so that the user can + estimate if there is still space free for records in the file. + */ + { + pgcache_page_no_t last_bitmap_page; + ulong blocks, bytes; + + last_bitmap_page= *last_page - *last_page % bitmap->pages_covered; + blocks= *last_page - last_bitmap_page; + bytes= (blocks * 3) / 8; /* 3 bit per page / 8 bits per byte */ + /* Size needs to be aligned on 6 */ + bytes/= 6; + bytes*= 6; + bitmap->last_bitmap_page= last_bitmap_page; + bitmap->last_total_size= bytes; + *last_page= ((last_bitmap_page + bytes*8/3)); + } + + /* Restore first_bitmap_with_space if it's resonable */ + if (first_bitmap_with_space <= (share->state.state.data_file_length / + share->block_size)) + share->state.first_bitmap_with_space= first_bitmap_with_space; return 0; } @@ -616,7 +642,7 @@ void _ma_bitmap_delete_all(MARIA_SHARE *share) bzero(bitmap->map, bitmap->block_size); bitmap->changed= 1; bitmap->page= 0; - bitmap->used_size= bitmap->total_size; + bitmap->used_size= bitmap->total_size= bitmap->max_total_size; } DBUG_VOID_RETURN; } @@ -646,13 +672,20 @@ void _ma_bitmap_reset_cache(MARIA_SHARE *share) We can't read a page yet, as in some case we don't have an active page cache yet. Pretend we have a dummy, full and not changed bitmap page in memory. + + We set bitmap->page to a value so that if we use it in + move_to_next_bitmap() it will point to page 0. + (This can only happen if writing to a bitmap page fails) */ - bitmap->page= ~(ulonglong) 0; - bitmap->used_size= bitmap->total_size; + bitmap->page= ((pgcache_page_no_t) 0) - bitmap->pages_covered; + bitmap->used_size= bitmap->total_size= bitmap->max_total_size; bfill(bitmap->map, share->block_size, 255); #ifndef DBUG_OFF memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); #endif + + /* Start scanning for free space from start of file */ + share->state.first_bitmap_with_space = 0; } } @@ -920,6 +953,20 @@ void _ma_get_bitmap_description(MARIA_FILE_BITMAP *bitmap, } +/* + Adjust bitmap->total_size to not go over max_data_file_size +*/ + +static void adjust_total_size(MARIA_HA *info, pgcache_page_no_t page) +{ + MARIA_FILE_BITMAP *bitmap= &info->s->bitmap; + + if (page < bitmap->last_bitmap_page) + bitmap->total_size= bitmap->max_total_size; /* Use all bits in bitmap */ + else + bitmap->total_size= bitmap->last_total_size; +} + /*************************************************************************** Reading & writing bitmap pages ***************************************************************************/ @@ -956,12 +1003,16 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, DBUG_ASSERT(!bitmap->changed); bitmap->page= page; - if (((page + 1) * bitmap->block_size) > share->state.state.data_file_length) + if ((page + 1) * bitmap->block_size > share->state.state.data_file_length) { /* Inexistent or half-created page */ res= _ma_bitmap_create_missing(info, bitmap, page); + if (!res) + adjust_total_size(info, page); DBUG_RETURN(res); } + + adjust_total_size(info, page); bitmap->used_size= bitmap->total_size; DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size); res= pagecache_read(share->pagecache, @@ -1045,14 +1096,18 @@ static my_bool move_to_next_bitmap(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap) MARIA_STATE_INFO *state= &info->s->state; DBUG_ENTER("move_to_next_bitmap"); - if (state->first_bitmap_with_space != ~(ulonglong) 0 && + if (state->first_bitmap_with_space != ~(pgcache_page_no_t) 0 && state->first_bitmap_with_space != page) { page= state->first_bitmap_with_space; - state->first_bitmap_with_space= ~(ulonglong) 0; + state->first_bitmap_with_space= ~(pgcache_page_no_t) 0; + DBUG_ASSERT(page % bitmap->pages_covered == 0); } else + { page+= bitmap->pages_covered; + DBUG_ASSERT(page % bitmap->pages_covered == 0); + } DBUG_RETURN(_ma_change_bitmap_page(info, bitmap, page)); } @@ -2933,6 +2988,11 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, /* First (in offset order) bitmap page to create */ if (data_file_length < block_size) goto err; /* corrupted, should have first bitmap page */ + if (page * block_size >= share->base.max_data_file_length) + { + my_errno= HA_ERR_RECORD_FILE_FULL; + goto err; + } from= (data_file_length / block_size - 1) / bitmap->pages_covered + 1; from*= bitmap->pages_covered; diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c index cb032a243ea..6f42aa29b03 100644 --- a/storage/maria/ma_blockrec.c +++ b/storage/maria/ma_blockrec.c @@ -414,14 +414,29 @@ void _ma_init_block_record_data(void) my_bool _ma_once_init_block_record(MARIA_SHARE *share, File data_file) { + my_bool res; + pgcache_page_no_t last_page; + + /* + First calculate the max file length with can have with a pointer of size + rec_reflength. - share->base.max_data_file_length= - (((ulonglong) 1 << ((share->base.rec_reflength-1)*8))-1) * - share->block_size; + The 'rec_reflength - 1' is because one byte is used for row + position withing the page. + The /2 comes from _ma_transaction_recpos_to_keypos() where we use + the lowest bit to mark if there is a transid following the rownr. + */ + last_page= ((ulonglong) 1 << ((share->base.rec_reflength-1)*8))/2; + if (!last_page) /* Overflow; set max size */ + last_page= ~(pgcache_page_no_t) 0; + + res= _ma_bitmap_init(share, data_file, &last_page); + share->base.max_data_file_length= _ma_safe_mul(last_page + 1, + share->block_size); #if SIZEOF_OFF_T == 4 - set_if_smaller(share->base.max_data_file_length, INT_MAX32); + set_if_smaller(share->base.max_data_file_length, INT_MAX32); #endif - return _ma_bitmap_init(share, data_file); + return res; } @@ -5170,7 +5185,7 @@ my_bool _ma_scan_init_block_record(MARIA_HA *info) (uchar *) my_malloc(share->block_size * 2, MYF(MY_WME)))))) DBUG_RETURN(1); info->scan.page_buff= info->scan.bitmap_buff + share->block_size; - info->scan.bitmap_end= info->scan.bitmap_buff + share->bitmap.total_size; + info->scan.bitmap_end= info->scan.bitmap_buff + share->bitmap.max_total_size; /* Set scan variables to get _ma_scan_block() to start with reading bitmap */ info->scan.number_of_rows= 0; diff --git a/storage/maria/ma_blockrec.h b/storage/maria/ma_blockrec.h index 3201998e31c..2a323d16ffa 100644 --- a/storage/maria/ma_blockrec.h +++ b/storage/maria/ma_blockrec.h @@ -187,7 +187,8 @@ maria_page_get_lsn(uchar *page, pgcache_page_no_t page_no, uchar* data_ptr); /* ma_bitmap.c */ extern const char *bits_to_txt[]; -my_bool _ma_bitmap_init(MARIA_SHARE *share, File file); +my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, + pgcache_page_no_t *last_page); my_bool _ma_bitmap_end(MARIA_SHARE *share); my_bool _ma_bitmap_flush(MARIA_SHARE *share); my_bool _ma_bitmap_flush_all(MARIA_SHARE *share); diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index f95b3184e29..b2112719c5e 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -405,26 +405,34 @@ int maria_chk_size(HA_CHECK *param, register MARIA_HA *info) size= my_seek(share->kfile.file, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)); if ((skr=(my_off_t) share->state.state.key_file_length) != size) { - /* Don't give error if file generated by mariapack */ + /* Don't give error if file generated by maria_pack */ if (skr > size && maria_is_any_key_active(share->state.key_map)) { error=1; _ma_check_print_error(param, "Size of indexfile is: %-8s Should be: %s", llstr(size,buff), llstr(skr,buff2)); + share->state.state.key_file_length= size; } else if (!(param->testflag & T_VERY_SILENT)) _ma_check_print_warning(param, "Size of indexfile is: %-8s Should be: %s", llstr(size,buff), llstr(skr,buff2)); } - if (!(param->testflag & T_VERY_SILENT) && - ! (share->options & HA_OPTION_COMPRESS_RECORD) && - ulonglong2double(share->state.state.key_file_length) > - ulonglong2double(share->base.margin_key_file_length)*0.9) + if (size > share->base.max_key_file_length) + { + _ma_check_print_warning(param, + "Size of indexfile is: %-8s which is bigger than max indexfile size: %s", + ullstr(size,buff), + ullstr(share->base.max_key_file_length, buff2)); + } + else if (!(param->testflag & T_VERY_SILENT) && + ! (share->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(share->state.state.key_file_length) > + ulonglong2double(share->base.margin_key_file_length)*0.9) _ma_check_print_warning(param,"Keyfile is almost full, %10s of %10s used", llstr(share->state.state.key_file_length,buff), - llstr(share->base.max_key_file_length-1,buff)); + llstr(share->base.max_key_file_length,buff)); size= my_seek(info->dfile.file, 0L, MY_SEEK_END, MYF(0)); skr=(my_off_t) share->state.state.data_file_length; @@ -437,9 +445,9 @@ int maria_chk_size(HA_CHECK *param, register MARIA_HA *info) #endif if (skr != size) { + share->state.state.data_file_length=size; /* Skip other errors */ if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN) { - share->state.state.data_file_length=size; /* Skip other errors */ error=1; _ma_check_print_error(param,"Size of datafile is: %-9s Should be: %s", llstr(size,buff), llstr(skr,buff2)); @@ -452,13 +460,19 @@ int maria_chk_size(HA_CHECK *param, register MARIA_HA *info) llstr(size,buff), llstr(skr,buff2)); } } - if (!(param->testflag & T_VERY_SILENT) && - !(share->options & HA_OPTION_COMPRESS_RECORD) && - ulonglong2double(share->state.state.data_file_length) > - (ulonglong2double(share->base.max_data_file_length)*0.9)) + if (size > share->base.max_data_file_length) + { + _ma_check_print_warning(param, + "Size of datafile is: %-8s which is bigger than max datafile size: %s", + ullstr(size,buff), + ullstr(share->base.max_data_file_length, buff2)); + } else if (!(param->testflag & T_VERY_SILENT) && + !(share->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(share->state.state.data_file_length) > + (ulonglong2double(share->base.max_data_file_length)*0.9)) _ma_check_print_warning(param, "Datafile is almost full, %10s of %10s used", llstr(share->state.state.data_file_length,buff), - llstr(share->base.max_data_file_length-1,buff2)); + llstr(share->base.max_data_file_length,buff2)); DBUG_RETURN(error); } /* maria_chk_size */ @@ -1967,8 +1981,8 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend, data= bitmap_buff + offset_page / 8; bitmap_pattern= uint2korr(data); if (((bitmap_pattern >> offset)) || - (data + 2 < bitmap_buff + share->bitmap.total_size && - _ma_check_if_zero(data+2, bitmap_buff + share->bitmap.total_size - + (data + 2 < bitmap_buff + share->bitmap.max_total_size && + _ma_check_if_zero(data+2, bitmap_buff + share->bitmap.max_total_size - data - 2))) { ulonglong bitmap_page; @@ -2074,23 +2088,23 @@ int maria_chk_data_link(HA_CHECK *param, MARIA_HA *info, my_bool extend) llstr(share->state.state.records,llbuff2)); error=1; } - else if (param->record_checksum && + if (param->record_checksum && param->record_checksum != param->tmp_record_checksum) { _ma_check_print_error(param, "Key pointers and record positions doesn't match"); error=1; } - else if (param->glob_crc != share->state.state.checksum && - (share->options & - (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))) + if (param->glob_crc != share->state.state.checksum && + (share->options & + (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))) { _ma_check_print_warning(param, "Record checksum is not the same as checksum " "stored in the index file"); error=1; } - else if (!extend) + if (!extend) { uint key; for (key=0 ; key < share->base.keys; key++) @@ -5256,7 +5270,10 @@ int _ma_sort_write_record(MARIA_SORT_PARAM *sort_param) if ((sort_param->current_filepos= (*share->write_record_init)(info, sort_param->record)) == HA_OFFSET_ERROR) + { + _ma_check_print_error(param, "%d when writing to datafile", my_errno); DBUG_RETURN(1); + } /* Pointer to end of file */ sort_param->filepos= share->state.state.data_file_length; break; @@ -5876,6 +5893,9 @@ int maria_recreate_table(HA_CHECK *param, MARIA_HA **org_info, char *filename) MARIA_CREATE_INFO create_info; DBUG_ENTER("maria_recreate_table"); + if ((!(param->testflag & T_SILENT))) + printf("Recreating table '%s'\n", param->isam_file_name); + error=1; /* Default error */ info= **org_info; status_info= (*org_info)->state[0]; diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index 6967a47a023..4d52a13453d 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -319,7 +319,15 @@ int maria_create(const char *name, enum data_file_type datafile_type, (~(ulonglong) 0)/ci->max_rows < (ulonglong) pack_reclength) ci->data_file_length= ~(ulonglong) 0; else - ci->data_file_length=(ulonglong) ci->max_rows*pack_reclength; + { + ci->data_file_length= _ma_safe_mul(ci->max_rows, pack_reclength); + if (datafile_type == BLOCK_RECORD) + { + /* Assume that blocks are only half full (very pessimistic!) */ + ci->data_file_length= _ma_safe_mul(ci->data_file_length, 2); + set_if_bigger(ci->data_file_length, maria_block_size*2); + } + } } else if (!ci->max_rows) { @@ -331,7 +339,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, ulonglong data_file_length= ci->data_file_length; if (!data_file_length) data_file_length= ((((ulonglong) 1 << ((BLOCK_RECORD_POINTER_SIZE-1) * - 8)) -1) * maria_block_size); + 8))/2 -1) * maria_block_size); if (rows_per_page > 0) { set_if_smaller(rows_per_page, MAX_ROWS_PER_PAGE); @@ -353,11 +361,11 @@ int maria_create(const char *name, enum data_file_type datafile_type, { /* The + 1 is for record position withing page - The / 2 is because we need one bit for knowing if there is transid's + The * 2 is because we need one bit for knowing if there is transid's after the row pointer */ pointer= maria_get_pointer_length((ci->data_file_length / - (maria_block_size * 2)), 3) + 1; + maria_block_size) * 2, 3) + 1; set_if_smaller(pointer, BLOCK_RECORD_POINTER_SIZE); if (!max_rows) diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index 7821b474988..e1f561b420a 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -502,7 +502,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) (ulonglong) 1 << (share->base.rec_reflength*8))-1); max_key_file_length= - _ma_safe_mul(maria_block_size, + _ma_safe_mul(share->base.block_size, ((ulonglong) 1 << (share->base.key_reflength*8))-1); #if SIZEOF_OFF_T == 4 set_if_smaller(max_data_file_length, INT_MAX32); @@ -815,7 +815,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) share->base.margin_key_file_length=(share->base.max_key_file_length - (keys ? MARIA_INDEX_BLOCK_MARGIN * share->block_size * keys : 0)); - share->block_size= share->base.block_size; + my_free(disk_cache, MYF(0)); _ma_setup_functions(share); if ((*share->once_init)(share, info.dfile.file)) diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index 7c0995da1a1..5c0ba13b2ab 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -1553,8 +1553,8 @@ static void descript(HA_CHECK *param, register MARIA_HA *info, char *name) if (share->base.max_data_file_length != HA_OFFSET_ERROR || share->base.max_key_file_length != HA_OFFSET_ERROR) printf("Max datafile length: %16s Max keyfile length: %18s\n", - llstr(share->base.max_data_file_length-1,llbuff), - llstr(share->base.max_key_file_length-1,llbuff2)); + ullstr(share->base.max_data_file_length,llbuff), + ullstr(share->base.max_key_file_length,llbuff2)); } } printf("Block_size: %16d\n",(int) share->block_size); diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index c50e95b6511..41501b8eb74 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -245,9 +245,10 @@ typedef struct st_maria_file_bitmap { uchar *map; pgcache_page_no_t page; /* Page number for current bitmap */ - uint used_size; /* Size of bitmap head that is not 0 */ + pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ my_bool changed; /* 1 if page needs to be written */ my_bool changed_not_flushed; /* 1 if some bitmap is not flushed */ + uint used_size; /* Size of bitmap head that is not 0 */ uint flush_all_requested; /**< If _ma_bitmap_flush_all waiting */ uint waiting_for_flush_all_requested; /* If someone is waiting for above */ uint non_flushable; /**< 0 if bitmap and log are in sync */ @@ -261,6 +262,8 @@ typedef struct st_maria_file_bitmap /* Constants, allocated when initiating bitmaps */ uint sizes[8]; /* Size per bit combination */ uint total_size; /* Total usable size of bitmap page */ + uint max_total_size; /* Max value for total_size */ + uint last_total_size; /* Size of bitmap on last_bitmap_page */ uint block_size; /* Block size of file */ ulong pages_covered; /* Pages covered by bitmap + 1 */ DYNAMIC_ARRAY pinned_pages; /**< not-yet-flushable bitmap pages */ |